2024-02-07T12:09:14.264 INFO:root:teuthology version: 0.0.1.dev256+gd9fdb22 2024-02-07T12:09:14.264 DEBUG:teuthology.run:Teuthology command: teuthology --name gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi --verbose --owner scheduled_gabrioux@teuthology --description orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} --archive /home/teuthworker/archive/gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi/7548733 -- /home/teuthworker/archive/gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi/7548733/orig.config.yaml 2024-02-07T12:09:14.314 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-02-07T12:09:14.395 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi/7548733 branch: wip-guits-testing-4-2024-02-05-1854 description: orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} email: gabrioux@ibm.com first_in_suite: false job_id: '7548733' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi name: gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi no_nested_subset: false nuke-on-error: true os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-guits-testing-4-2024-02-05-1854 ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mgr/cephadm/use_agent: false mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 44458ccedb18241cc2e17f10cdab3d1385399261 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 44458ccedb18241cc2e17f10cdab3d1385399261 workunit: branch: wip-guits-testing-4-2024-02-05-1854 sha1: 44458ccedb18241cc2e17f10cdab3d1385399261 owner: scheduled_gabrioux@teuthology priority: 80 repo: https://git.ceph.com/ceph-ci.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 1701 sha1: 44458ccedb18241cc2e17f10cdab3d1385399261 sleep_before_teardown: 0 subset: 1/8 suite: orch:cephadm suite_branch: wip-guits-testing-4-2024-02-05-1854 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_44458ccedb18241cc2e17f10cdab3d1385399261/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 44458ccedb18241cc2e17f10cdab3d1385399261 targets: smithi019.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPwFE8Culvu+H8Do8d2TM4Slqxtg4MbUMrg206NbiNC7YeWjtzomiadrQcQ6dXFYEm5ug56l+DV6sOh9A8kd1wU= smithi172.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFWm+D0M+O6W7kbCyB1ja37/KkmK5mzxLwfjNocn+bQhp3vsJJKP9RHtk4u1YFJWWddksiKfNuLyjszgr90E7J8= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 - cephadm.shell: env: - sha1 mon.a: - radosgw-admin realm create --rgw-realm=r --default - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default - radosgw-admin period update --rgw-realm=r --commit - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000 - ceph osd pool create foo - rbd pool init foo - ceph orch apply iscsi foo u p - sleep 180 - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph orch ps - ceph versions - ceph -s - ceph orch ls - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 2' - ceph mgr fail - sleep 180 - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph health detail - ceph -s - ceph mgr fail - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 1' - ceph mgr fail - sleep 180 - ceph orch ps - ceph orch ls - ceph versions - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph versions | jq -e '.mgr | length == 1' - ceph versions | jq -e '.mgr | keys' | grep $sha1 - ceph versions | jq -e '.overall | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 1' - ceph versions | jq -e '.mon | keys' | grep $sha1 - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 1' - ceph versions | jq -e '.osd | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.rgw | length == 1' - ceph versions | jq -e '.rgw | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 mon.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph orch upgrade status - ceph health detail - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - ceph orch ls | grep '^osd ' - cephadm.shell: mon.a: - ceph orch upgrade ls - ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0 - ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: d9fdb2209e15b39d9f061fd85399f352ce0f0894 timestamp: 2024-02-06_08:19:47 tube: smithi user: gabrioux verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-02-07T12:09:14.395 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_44458ccedb18241cc2e17f10cdab3d1385399261/qa; will attempt to use it 2024-02-07T12:09:14.396 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_44458ccedb18241cc2e17f10cdab3d1385399261/qa/tasks 2024-02-07T12:09:14.396 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-02-07T12:09:14.398 INFO:teuthology.task.internal:Checking packages... 2024-02-07T12:09:14.431 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '44458ccedb18241cc2e17f10cdab3d1385399261' 2024-02-07T12:09:14.431 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-02-07T12:09:14.431 INFO:teuthology.packaging:ref: None 2024-02-07T12:09:14.431 INFO:teuthology.packaging:tag: None 2024-02-07T12:09:14.431 INFO:teuthology.packaging:branch: wip-guits-testing-4-2024-02-05-1854 2024-02-07T12:09:14.431 INFO:teuthology.packaging:sha1: 44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:09:14.431 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-guits-testing-4-2024-02-05-1854 2024-02-07T12:09:14.574 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-1252.g44458cce 2024-02-07T12:09:14.575 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-02-07T12:09:14.582 INFO:teuthology.task.internal:no buildpackages task found 2024-02-07T12:09:14.583 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-02-07T12:09:14.589 INFO:teuthology.task.internal:Saving configuration 2024-02-07T12:09:14.607 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-02-07T12:09:14.614 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-02-07T12:09:14.641 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi019.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi/7548733', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-02-07 12:04:22.370021', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPwFE8Culvu+H8Do8d2TM4Slqxtg4MbUMrg206NbiNC7YeWjtzomiadrQcQ6dXFYEm5ug56l+DV6sOh9A8kd1wU='} 2024-02-07T12:09:14.663 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi172.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi/7548733', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-02-07 12:04:22.371828', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFWm+D0M+O6W7kbCyB1ja37/KkmK5mzxLwfjNocn+bQhp3vsJJKP9RHtk4u1YFJWWddksiKfNuLyjszgr90E7J8='} 2024-02-07T12:09:14.663 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-02-07T12:09:14.670 INFO:teuthology.task.internal:roles: ubuntu@smithi019.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-02-07T12:09:14.670 INFO:teuthology.task.internal:roles: ubuntu@smithi172.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-02-07T12:09:14.670 INFO:teuthology.run_tasks:Running task console_log... 2024-02-07T12:09:14.745 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fa25ceec0d0>, signals=[15]) 2024-02-07T12:09:14.745 INFO:teuthology.run_tasks:Running task internal.connect... 2024-02-07T12:09:14.752 INFO:teuthology.task.internal:Opening connections... 2024-02-07T12:09:14.752 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi019.front.sepia.ceph.com 2024-02-07T12:09:14.754 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi019.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-07T12:09:14.886 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi172.front.sepia.ceph.com 2024-02-07T12:09:14.887 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi172.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-07T12:09:15.021 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-02-07T12:09:15.028 DEBUG:teuthology.orchestra.run.smithi019:> uname -m 2024-02-07T12:09:15.046 INFO:teuthology.orchestra.run.smithi019.stdout:x86_64 2024-02-07T12:09:15.046 DEBUG:teuthology.orchestra.run.smithi019:> cat /etc/os-release 2024-02-07T12:09:15.104 INFO:teuthology.orchestra.run.smithi019.stdout:NAME="CentOS Stream" 2024-02-07T12:09:15.105 INFO:teuthology.orchestra.run.smithi019.stdout:VERSION="9" 2024-02-07T12:09:15.105 INFO:teuthology.orchestra.run.smithi019.stdout:ID="centos" 2024-02-07T12:09:15.105 INFO:teuthology.orchestra.run.smithi019.stdout:ID_LIKE="rhel fedora" 2024-02-07T12:09:15.105 INFO:teuthology.orchestra.run.smithi019.stdout:VERSION_ID="9" 2024-02-07T12:09:15.105 INFO:teuthology.orchestra.run.smithi019.stdout:PLATFORM_ID="platform:el9" 2024-02-07T12:09:15.105 INFO:teuthology.orchestra.run.smithi019.stdout:PRETTY_NAME="CentOS Stream 9" 2024-02-07T12:09:15.105 INFO:teuthology.orchestra.run.smithi019.stdout:ANSI_COLOR="0;31" 2024-02-07T12:09:15.105 INFO:teuthology.orchestra.run.smithi019.stdout:LOGO="fedora-logo-icon" 2024-02-07T12:09:15.105 INFO:teuthology.orchestra.run.smithi019.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-02-07T12:09:15.105 INFO:teuthology.orchestra.run.smithi019.stdout:HOME_URL="https://centos.org/" 2024-02-07T12:09:15.105 INFO:teuthology.orchestra.run.smithi019.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-02-07T12:09:15.105 INFO:teuthology.orchestra.run.smithi019.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-02-07T12:09:15.105 INFO:teuthology.orchestra.run.smithi019.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-02-07T12:09:15.106 INFO:teuthology.lock.ops:Updating smithi019.front.sepia.ceph.com on lock server 2024-02-07T12:09:15.151 DEBUG:teuthology.orchestra.run.smithi172:> uname -m 2024-02-07T12:09:15.169 INFO:teuthology.orchestra.run.smithi172.stdout:x86_64 2024-02-07T12:09:15.169 DEBUG:teuthology.orchestra.run.smithi172:> cat /etc/os-release 2024-02-07T12:09:15.227 INFO:teuthology.orchestra.run.smithi172.stdout:NAME="CentOS Stream" 2024-02-07T12:09:15.227 INFO:teuthology.orchestra.run.smithi172.stdout:VERSION="9" 2024-02-07T12:09:15.227 INFO:teuthology.orchestra.run.smithi172.stdout:ID="centos" 2024-02-07T12:09:15.227 INFO:teuthology.orchestra.run.smithi172.stdout:ID_LIKE="rhel fedora" 2024-02-07T12:09:15.227 INFO:teuthology.orchestra.run.smithi172.stdout:VERSION_ID="9" 2024-02-07T12:09:15.227 INFO:teuthology.orchestra.run.smithi172.stdout:PLATFORM_ID="platform:el9" 2024-02-07T12:09:15.227 INFO:teuthology.orchestra.run.smithi172.stdout:PRETTY_NAME="CentOS Stream 9" 2024-02-07T12:09:15.227 INFO:teuthology.orchestra.run.smithi172.stdout:ANSI_COLOR="0;31" 2024-02-07T12:09:15.227 INFO:teuthology.orchestra.run.smithi172.stdout:LOGO="fedora-logo-icon" 2024-02-07T12:09:15.228 INFO:teuthology.orchestra.run.smithi172.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-02-07T12:09:15.228 INFO:teuthology.orchestra.run.smithi172.stdout:HOME_URL="https://centos.org/" 2024-02-07T12:09:15.228 INFO:teuthology.orchestra.run.smithi172.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-02-07T12:09:15.228 INFO:teuthology.orchestra.run.smithi172.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-02-07T12:09:15.228 INFO:teuthology.orchestra.run.smithi172.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-02-07T12:09:15.228 INFO:teuthology.lock.ops:Updating smithi172.front.sepia.ceph.com on lock server 2024-02-07T12:09:15.258 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-02-07T12:09:15.268 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-02-07T12:09:15.274 INFO:teuthology.task.internal:Checking for old test directory... 2024-02-07T12:09:15.274 DEBUG:teuthology.orchestra.run.smithi019:> test '!' -e /home/ubuntu/cephtest 2024-02-07T12:09:15.277 DEBUG:teuthology.orchestra.run.smithi172:> test '!' -e /home/ubuntu/cephtest 2024-02-07T12:09:15.293 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-02-07T12:09:15.298 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-02-07T12:09:15.298 DEBUG:teuthology.orchestra.run.smithi019:> test -z $(ls -A /var/lib/ceph) 2024-02-07T12:09:15.334 DEBUG:teuthology.orchestra.run.smithi172:> test -z $(ls -A /var/lib/ceph) 2024-02-07T12:09:15.362 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-02-07T12:09:15.416 INFO:teuthology.run_tasks:Running task kernel... 2024-02-07T12:09:15.430 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2024-02-07T12:09:15.430 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2024-02-07T12:09:15.431 DEBUG:teuthology.orchestra.run.smithi019:> test -f /run/.containerenv -o -f /.dockerenv 2024-02-07T12:09:15.431 DEBUG:teuthology.orchestra.run.smithi172:> test -f /run/.containerenv -o -f /.dockerenv 2024-02-07T12:09:15.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-02-07T12:09:15.448 DEBUG:teuthology.orchestra.run.smithi172:> uname -r 2024-02-07T12:09:15.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-02-07T12:09:15.449 DEBUG:teuthology.orchestra.run.smithi019:> uname -r 2024-02-07T12:09:15.506 INFO:teuthology.orchestra.run.smithi172.stdout:5.14.0-361.el9.x86_64 2024-02-07T12:09:15.506 INFO:teuthology.task.kernel:Running kernel on smithi172: 5.14.0-361.el9.x86_64 2024-02-07T12:09:15.506 DEBUG:teuthology.orchestra.run.smithi172:> sudo yum install -y kernel 2024-02-07T12:09:15.507 INFO:teuthology.orchestra.run.smithi019.stdout:5.14.0-361.el9.x86_64 2024-02-07T12:09:15.507 INFO:teuthology.task.kernel:Running kernel on smithi019: 5.14.0-361.el9.x86_64 2024-02-07T12:09:15.507 DEBUG:teuthology.orchestra.run.smithi019:> sudo yum install -y kernel 2024-02-07T12:09:18.049 INFO:teuthology.orchestra.run.smithi172.stdout:CentOS-9 - AppStream 8.4 MB/s | 6.7 MB 00:00 2024-02-07T12:09:18.912 INFO:teuthology.orchestra.run.smithi019.stdout:CentOS-9 - AppStream 4.3 MB/s | 6.7 MB 00:01 2024-02-07T12:09:19.649 INFO:teuthology.orchestra.run.smithi172.stdout:CentOS-9 - BaseOS 4.1 MB/s | 2.0 MB 00:00 2024-02-07T12:09:20.736 INFO:teuthology.orchestra.run.smithi019.stdout:CentOS-9 - BaseOS 3.0 MB/s | 2.0 MB 00:00 2024-02-07T12:09:20.981 INFO:teuthology.orchestra.run.smithi172.stdout:CentOS Stream 9 - BaseOS 8.0 MB/s | 8.0 MB 00:01 2024-02-07T12:09:25.068 INFO:teuthology.orchestra.run.smithi172.stdout:CentOS Stream 9 - AppStream 5.7 MB/s | 19 MB 00:03 2024-02-07T12:09:27.226 INFO:teuthology.orchestra.run.smithi019.stdout:CentOS Stream 9 - BaseOS 1.3 MB/s | 8.0 MB 00:06 2024-02-07T12:09:28.933 INFO:teuthology.orchestra.run.smithi172.stdout:CentOS Stream 9 - CRB 15 MB/s | 5.9 MB 00:00 2024-02-07T12:09:30.484 INFO:teuthology.orchestra.run.smithi172.stdout:CentOS Stream 9 - Extras packages 33 kB/s | 15 kB 00:00 2024-02-07T12:09:30.744 INFO:teuthology.orchestra.run.smithi019.stdout:CentOS Stream 9 - AppStream 7.8 MB/s | 19 MB 00:02 2024-02-07T12:09:35.994 INFO:teuthology.orchestra.run.smithi019.stdout:CentOS Stream 9 - CRB 3.7 MB/s | 5.9 MB 00:01 2024-02-07T12:09:37.587 INFO:teuthology.orchestra.run.smithi019.stdout:CentOS Stream 9 - Extras packages 34 kB/s | 15 kB 00:00 2024-02-07T12:09:38.647 INFO:teuthology.orchestra.run.smithi019.stdout:Extra Packages for Enterprise Linux 22 MB/s | 20 MB 00:00 2024-02-07T12:09:42.152 INFO:teuthology.orchestra.run.smithi172.stdout:Extra Packages for Enterprise Linux 1.8 MB/s | 20 MB 00:11 2024-02-07T12:09:43.713 INFO:teuthology.orchestra.run.smithi019.stdout:lab-extras 32 kB/s | 1.7 kB 00:00 2024-02-07T12:09:45.987 INFO:teuthology.orchestra.run.smithi019.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-02-07T12:09:45.987 INFO:teuthology.orchestra.run.smithi019.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-02-07T12:09:46.093 INFO:teuthology.orchestra.run.smithi019.stdout:Dependencies resolved. 2024-02-07T12:09:46.112 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2024-02-07T12:09:46.113 INFO:teuthology.orchestra.run.smithi019.stdout: Package Arch Version Repository Size 2024-02-07T12:09:46.113 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2024-02-07T12:09:46.113 INFO:teuthology.orchestra.run.smithi019.stdout:Installing: 2024-02-07T12:09:46.113 INFO:teuthology.orchestra.run.smithi019.stdout: kernel x86_64 5.14.0-418.el9 CentOS-BaseOS 6.0 M 2024-02-07T12:09:46.113 INFO:teuthology.orchestra.run.smithi019.stdout:Installing dependencies: 2024-02-07T12:09:46.113 INFO:teuthology.orchestra.run.smithi019.stdout: kernel-core x86_64 5.14.0-418.el9 CentOS-BaseOS 21 M 2024-02-07T12:09:46.113 INFO:teuthology.orchestra.run.smithi019.stdout: kernel-modules x86_64 5.14.0-418.el9 CentOS-BaseOS 39 M 2024-02-07T12:09:46.113 INFO:teuthology.orchestra.run.smithi019.stdout: kernel-modules-core x86_64 5.14.0-418.el9 CentOS-BaseOS 33 M 2024-02-07T12:09:46.113 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:09:46.113 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction Summary 2024-02-07T12:09:46.113 INFO:teuthology.orchestra.run.smithi019.stdout:================================================================================ 2024-02-07T12:09:46.113 INFO:teuthology.orchestra.run.smithi019.stdout:Install 4 Packages 2024-02-07T12:09:46.113 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:09:46.113 INFO:teuthology.orchestra.run.smithi019.stdout:Total download size: 99 M 2024-02-07T12:09:46.113 INFO:teuthology.orchestra.run.smithi019.stdout:Installed size: 122 M 2024-02-07T12:09:46.114 INFO:teuthology.orchestra.run.smithi019.stdout:Downloading Packages: 2024-02-07T12:09:46.473 INFO:teuthology.orchestra.run.smithi019.stdout:(1/4): kernel-5.14.0-418.el9.x86_64.rpm 16 MB/s | 6.0 MB 00:00 2024-02-07T12:09:46.984 INFO:teuthology.orchestra.run.smithi172.stdout:lab-extras 29 kB/s | 1.7 kB 00:00 2024-02-07T12:09:48.023 INFO:teuthology.orchestra.run.smithi019.stdout:(2/4): kernel-modules-5.14.0-418.el9.x86_64.rpm 20 MB/s | 39 MB 00:01 2024-02-07T12:09:48.232 INFO:teuthology.orchestra.run.smithi019.stdout:(3/4): kernel-core-5.14.0-418.el9.x86_64.rpm 9.8 MB/s | 21 MB 00:02 2024-02-07T12:09:48.932 INFO:teuthology.orchestra.run.smithi019.stdout:(4/4): kernel-modules-core-5.14.0-418.el9.x86_6 14 MB/s | 33 MB 00:02 2024-02-07T12:09:48.935 INFO:teuthology.orchestra.run.smithi019.stdout:-------------------------------------------------------------------------------- 2024-02-07T12:09:48.935 INFO:teuthology.orchestra.run.smithi019.stdout:Total 35 MB/s | 99 MB 00:02 2024-02-07T12:09:48.935 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction check 2024-02-07T12:09:49.240 INFO:teuthology.orchestra.run.smithi172.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-02-07T12:09:49.240 INFO:teuthology.orchestra.run.smithi172.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-02-07T12:09:49.333 INFO:teuthology.orchestra.run.smithi172.stdout:Dependencies resolved. 2024-02-07T12:09:49.342 INFO:teuthology.orchestra.run.smithi172.stdout:================================================================================ 2024-02-07T12:09:49.342 INFO:teuthology.orchestra.run.smithi172.stdout: Package Arch Version Repository Size 2024-02-07T12:09:49.342 INFO:teuthology.orchestra.run.smithi172.stdout:================================================================================ 2024-02-07T12:09:49.342 INFO:teuthology.orchestra.run.smithi172.stdout:Installing: 2024-02-07T12:09:49.342 INFO:teuthology.orchestra.run.smithi172.stdout: kernel x86_64 5.14.0-418.el9 CentOS-BaseOS 6.0 M 2024-02-07T12:09:49.342 INFO:teuthology.orchestra.run.smithi172.stdout:Installing dependencies: 2024-02-07T12:09:49.342 INFO:teuthology.orchestra.run.smithi172.stdout: kernel-core x86_64 5.14.0-418.el9 CentOS-BaseOS 21 M 2024-02-07T12:09:49.342 INFO:teuthology.orchestra.run.smithi172.stdout: kernel-modules x86_64 5.14.0-418.el9 CentOS-BaseOS 39 M 2024-02-07T12:09:49.342 INFO:teuthology.orchestra.run.smithi172.stdout: kernel-modules-core x86_64 5.14.0-418.el9 CentOS-BaseOS 33 M 2024-02-07T12:09:49.343 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-02-07T12:09:49.343 INFO:teuthology.orchestra.run.smithi172.stdout:Transaction Summary 2024-02-07T12:09:49.343 INFO:teuthology.orchestra.run.smithi172.stdout:================================================================================ 2024-02-07T12:09:49.343 INFO:teuthology.orchestra.run.smithi172.stdout:Install 4 Packages 2024-02-07T12:09:49.343 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-02-07T12:09:49.343 INFO:teuthology.orchestra.run.smithi172.stdout:Total download size: 99 M 2024-02-07T12:09:49.343 INFO:teuthology.orchestra.run.smithi172.stdout:Installed size: 122 M 2024-02-07T12:09:49.344 INFO:teuthology.orchestra.run.smithi172.stdout:Downloading Packages: 2024-02-07T12:09:49.397 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction check succeeded. 2024-02-07T12:09:49.397 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction test 2024-02-07T12:09:49.776 INFO:teuthology.orchestra.run.smithi172.stdout:(1/4): kernel-5.14.0-418.el9.x86_64.rpm 14 MB/s | 6.0 MB 00:00 2024-02-07T12:09:50.268 INFO:teuthology.orchestra.run.smithi172.stdout:(2/4): kernel-core-5.14.0-418.el9.x86_64.rpm 23 MB/s | 21 MB 00:00 2024-02-07T12:09:50.476 INFO:teuthology.orchestra.run.smithi019.stdout:Transaction test succeeded. 2024-02-07T12:09:50.477 INFO:teuthology.orchestra.run.smithi019.stdout:Running transaction 2024-02-07T12:09:51.043 INFO:teuthology.orchestra.run.smithi172.stdout:(3/4): kernel-modules-5.14.0-418.el9.x86_64.rpm 23 MB/s | 39 MB 00:01 2024-02-07T12:09:51.477 INFO:teuthology.orchestra.run.smithi172.stdout:(4/4): kernel-modules-core-5.14.0-418.el9.x86_6 20 MB/s | 33 MB 00:01 2024-02-07T12:09:51.480 INFO:teuthology.orchestra.run.smithi172.stdout:-------------------------------------------------------------------------------- 2024-02-07T12:09:51.480 INFO:teuthology.orchestra.run.smithi172.stdout:Total 46 MB/s | 99 MB 00:02 2024-02-07T12:09:51.481 INFO:teuthology.orchestra.run.smithi172.stdout:Running transaction check 2024-02-07T12:09:51.853 INFO:teuthology.orchestra.run.smithi172.stdout:Transaction check succeeded. 2024-02-07T12:09:51.853 INFO:teuthology.orchestra.run.smithi172.stdout:Running transaction test 2024-02-07T12:09:52.858 INFO:teuthology.orchestra.run.smithi172.stdout:Transaction test succeeded. 2024-02-07T12:09:52.859 INFO:teuthology.orchestra.run.smithi172.stdout:Running transaction 2024-02-07T12:09:52.994 INFO:teuthology.orchestra.run.smithi019.stdout: Preparing : 1/1 2024-02-07T12:09:53.669 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : kernel-modules-core-5.14.0-418.el9.x86_64 1/4 2024-02-07T12:09:53.800 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : kernel-core-5.14.0-418.el9.x86_64 2/4 2024-02-07T12:09:55.234 INFO:teuthology.orchestra.run.smithi172.stdout: Preparing : 1/1 2024-02-07T12:09:55.871 INFO:teuthology.orchestra.run.smithi172.stdout: Installing : kernel-modules-core-5.14.0-418.el9.x86_64 1/4 2024-02-07T12:09:56.002 INFO:teuthology.orchestra.run.smithi172.stdout: Installing : kernel-core-5.14.0-418.el9.x86_64 2/4 2024-02-07T12:09:56.719 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: kernel-core-5.14.0-418.el9.x86_64 2/4 2024-02-07T12:09:56.901 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : kernel-modules-5.14.0-418.el9.x86_64 3/4 2024-02-07T12:09:58.736 INFO:teuthology.orchestra.run.smithi172.stdout: Running scriptlet: kernel-core-5.14.0-418.el9.x86_64 2/4 2024-02-07T12:09:58.907 INFO:teuthology.orchestra.run.smithi172.stdout: Installing : kernel-modules-5.14.0-418.el9.x86_64 3/4 2024-02-07T12:10:01.994 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: kernel-modules-5.14.0-418.el9.x86_64 3/4 2024-02-07T12:10:02.098 INFO:teuthology.orchestra.run.smithi019.stdout: Installing : kernel-5.14.0-418.el9.x86_64 4/4 2024-02-07T12:10:03.577 INFO:teuthology.orchestra.run.smithi172.stdout: Running scriptlet: kernel-modules-5.14.0-418.el9.x86_64 3/4 2024-02-07T12:10:03.769 INFO:teuthology.orchestra.run.smithi172.stdout: Installing : kernel-5.14.0-418.el9.x86_64 4/4 2024-02-07T12:10:07.220 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: kernel-modules-core-5.14.0-418.el9.x86_64 4/4 2024-02-07T12:10:08.502 INFO:teuthology.orchestra.run.smithi172.stdout: Running scriptlet: kernel-modules-core-5.14.0-418.el9.x86_64 4/4 2024-02-07T12:10:37.673 INFO:teuthology.orchestra.run.smithi172.stdout: Running scriptlet: kernel-core-5.14.0-418.el9.x86_64 4/4 2024-02-07T12:10:37.674 INFO:teuthology.orchestra.run.smithi172.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-02-07T12:10:37.674 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-02-07T12:10:37.745 INFO:teuthology.orchestra.run.smithi172.stdout: Running scriptlet: kernel-modules-5.14.0-418.el9.x86_64 4/4 2024-02-07T12:10:37.972 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: kernel-core-5.14.0-418.el9.x86_64 4/4 2024-02-07T12:10:37.972 INFO:teuthology.orchestra.run.smithi019.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-02-07T12:10:37.972 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:10:38.048 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: kernel-modules-5.14.0-418.el9.x86_64 4/4 2024-02-07T12:10:38.155 INFO:teuthology.orchestra.run.smithi172.stdout: Running scriptlet: kernel-5.14.0-418.el9.x86_64 4/4 2024-02-07T12:10:38.155 INFO:teuthology.orchestra.run.smithi172.stdout: Verifying : kernel-5.14.0-418.el9.x86_64 1/4 2024-02-07T12:10:38.156 INFO:teuthology.orchestra.run.smithi172.stdout: Verifying : kernel-core-5.14.0-418.el9.x86_64 2/4 2024-02-07T12:10:38.156 INFO:teuthology.orchestra.run.smithi172.stdout: Verifying : kernel-modules-5.14.0-418.el9.x86_64 3/4 2024-02-07T12:10:38.472 INFO:teuthology.orchestra.run.smithi019.stdout: Running scriptlet: kernel-5.14.0-418.el9.x86_64 4/4 2024-02-07T12:10:38.472 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : kernel-5.14.0-418.el9.x86_64 1/4 2024-02-07T12:10:38.472 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : kernel-core-5.14.0-418.el9.x86_64 2/4 2024-02-07T12:10:38.472 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : kernel-modules-5.14.0-418.el9.x86_64 3/4 2024-02-07T12:10:38.624 INFO:teuthology.orchestra.run.smithi172.stdout: Verifying : kernel-modules-core-5.14.0-418.el9.x86_64 4/4 2024-02-07T12:10:38.624 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-02-07T12:10:38.624 INFO:teuthology.orchestra.run.smithi172.stdout:Installed: 2024-02-07T12:10:38.624 INFO:teuthology.orchestra.run.smithi172.stdout: kernel-5.14.0-418.el9.x86_64 2024-02-07T12:10:38.624 INFO:teuthology.orchestra.run.smithi172.stdout: kernel-core-5.14.0-418.el9.x86_64 2024-02-07T12:10:38.624 INFO:teuthology.orchestra.run.smithi172.stdout: kernel-modules-5.14.0-418.el9.x86_64 2024-02-07T12:10:38.624 INFO:teuthology.orchestra.run.smithi172.stdout: kernel-modules-core-5.14.0-418.el9.x86_64 2024-02-07T12:10:38.624 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-02-07T12:10:38.624 INFO:teuthology.orchestra.run.smithi172.stdout:Complete! 2024-02-07T12:10:38.896 INFO:teuthology.orchestra.run.smithi019.stdout: Verifying : kernel-modules-core-5.14.0-418.el9.x86_64 4/4 2024-02-07T12:10:38.896 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:10:38.896 INFO:teuthology.orchestra.run.smithi019.stdout:Installed: 2024-02-07T12:10:38.896 INFO:teuthology.orchestra.run.smithi019.stdout: kernel-5.14.0-418.el9.x86_64 2024-02-07T12:10:38.896 INFO:teuthology.orchestra.run.smithi019.stdout: kernel-core-5.14.0-418.el9.x86_64 2024-02-07T12:10:38.896 INFO:teuthology.orchestra.run.smithi019.stdout: kernel-modules-5.14.0-418.el9.x86_64 2024-02-07T12:10:38.896 INFO:teuthology.orchestra.run.smithi019.stdout: kernel-modules-core-5.14.0-418.el9.x86_64 2024-02-07T12:10:38.896 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:10:38.896 INFO:teuthology.orchestra.run.smithi019.stdout:Complete! 2024-02-07T12:10:39.142 DEBUG:teuthology.orchestra.run.smithi172:> rpm -q kernel | sort -rV | head -n 1 2024-02-07T12:10:39.312 INFO:teuthology.orchestra.run.smithi172.stdout:kernel-5.14.0-418.el9.x86_64 2024-02-07T12:10:39.313 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-418.el9.x86_64 2024-02-07T12:10:39.313 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-418.el9.x86_64 2024-02-07T12:10:39.313 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-02-07T12:10:39.313 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-02-07T12:10:39.313 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-02-07T12:10:39.313 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi172.front.sepia.ceph.com, path=None, version=distro) 2024-02-07T12:10:39.313 DEBUG:teuthology.orchestra.run.smithi172:> rpm -q kernel | sort -rV | head -n 1 2024-02-07T12:10:39.380 DEBUG:teuthology.orchestra.run.smithi019:> rpm -q kernel | sort -rV | head -n 1 2024-02-07T12:10:39.420 INFO:teuthology.orchestra.run.smithi172.stdout:kernel-5.14.0-418.el9.x86_64 2024-02-07T12:10:39.420 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-418.el9.x86_64 2024-02-07T12:10:39.421 DEBUG:teuthology.orchestra.run.smithi172:> sudo rpm -qi grub2-tools 2024-02-07T12:10:39.457 INFO:teuthology.orchestra.run.smithi172.stdout:Name : grub2-tools 2024-02-07T12:10:39.457 INFO:teuthology.orchestra.run.smithi172.stdout:Epoch : 1 2024-02-07T12:10:39.457 INFO:teuthology.orchestra.run.smithi172.stdout:Version : 2.06 2024-02-07T12:10:39.458 INFO:teuthology.orchestra.run.smithi172.stdout:Release : 68.el9 2024-02-07T12:10:39.458 INFO:teuthology.orchestra.run.smithi172.stdout:Architecture: x86_64 2024-02-07T12:10:39.458 INFO:teuthology.orchestra.run.smithi172.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-02-07T12:10:39.458 INFO:teuthology.orchestra.run.smithi172.stdout:Group : Unspecified 2024-02-07T12:10:39.458 INFO:teuthology.orchestra.run.smithi172.stdout:Size : 8264254 2024-02-07T12:10:39.458 INFO:teuthology.orchestra.run.smithi172.stdout:License : GPLv3+ 2024-02-07T12:10:39.458 INFO:teuthology.orchestra.run.smithi172.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-02-07T12:10:39.458 INFO:teuthology.orchestra.run.smithi172.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-02-07T12:10:39.458 INFO:teuthology.orchestra.run.smithi172.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-02-07T12:10:39.458 INFO:teuthology.orchestra.run.smithi172.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-02-07T12:10:39.458 INFO:teuthology.orchestra.run.smithi172.stdout:Packager : builder@centos.org 2024-02-07T12:10:39.458 INFO:teuthology.orchestra.run.smithi172.stdout:Vendor : CentOS 2024-02-07T12:10:39.458 INFO:teuthology.orchestra.run.smithi172.stdout:URL : http://www.gnu.org/software/grub/ 2024-02-07T12:10:39.458 INFO:teuthology.orchestra.run.smithi172.stdout:Summary : Support tools for GRUB. 2024-02-07T12:10:39.459 INFO:teuthology.orchestra.run.smithi172.stdout:Description : 2024-02-07T12:10:39.459 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-02-07T12:10:39.459 INFO:teuthology.orchestra.run.smithi172.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-02-07T12:10:39.459 INFO:teuthology.orchestra.run.smithi172.stdout:customizable bootloader with modular architecture. It supports a rich 2024-02-07T12:10:39.459 INFO:teuthology.orchestra.run.smithi172.stdout:variety of kernel formats, file systems, computer architectures and 2024-02-07T12:10:39.459 INFO:teuthology.orchestra.run.smithi172.stdout:hardware devices. 2024-02-07T12:10:39.459 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-02-07T12:10:39.459 INFO:teuthology.orchestra.run.smithi172.stdout:This subpackage provides tools for support of all platforms. 2024-02-07T12:10:39.460 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-02-07T12:10:39.460 INFO:teuthology.task.kernel:Updating grub on smithi172 to boot 5.14.0-418.el9.x86_64 2024-02-07T12:10:39.460 DEBUG:teuthology.orchestra.run.smithi172:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-02-07T12:10:39.517 INFO:teuthology.orchestra.run.smithi019.stdout:kernel-5.14.0-418.el9.x86_64 2024-02-07T12:10:39.517 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-418.el9.x86_64 2024-02-07T12:10:39.517 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-418.el9.x86_64 2024-02-07T12:10:39.517 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-02-07T12:10:39.517 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-02-07T12:10:39.517 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-02-07T12:10:39.517 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi019.front.sepia.ceph.com, path=None, version=distro) 2024-02-07T12:10:39.517 DEBUG:teuthology.orchestra.run.smithi019:> rpm -q kernel | sort -rV | head -n 1 2024-02-07T12:10:39.631 INFO:teuthology.orchestra.run.smithi019.stdout:kernel-5.14.0-418.el9.x86_64 2024-02-07T12:10:39.631 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-418.el9.x86_64 2024-02-07T12:10:39.631 DEBUG:teuthology.orchestra.run.smithi019:> sudo rpm -qi grub2-tools 2024-02-07T12:10:39.676 INFO:teuthology.orchestra.run.smithi019.stdout:Name : grub2-tools 2024-02-07T12:10:39.677 INFO:teuthology.orchestra.run.smithi019.stdout:Epoch : 1 2024-02-07T12:10:39.677 INFO:teuthology.orchestra.run.smithi019.stdout:Version : 2.06 2024-02-07T12:10:39.677 INFO:teuthology.orchestra.run.smithi019.stdout:Release : 68.el9 2024-02-07T12:10:39.677 INFO:teuthology.orchestra.run.smithi019.stdout:Architecture: x86_64 2024-02-07T12:10:39.677 INFO:teuthology.orchestra.run.smithi019.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-02-07T12:10:39.677 INFO:teuthology.orchestra.run.smithi019.stdout:Group : Unspecified 2024-02-07T12:10:39.677 INFO:teuthology.orchestra.run.smithi019.stdout:Size : 8264254 2024-02-07T12:10:39.677 INFO:teuthology.orchestra.run.smithi019.stdout:License : GPLv3+ 2024-02-07T12:10:39.677 INFO:teuthology.orchestra.run.smithi019.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-02-07T12:10:39.677 INFO:teuthology.orchestra.run.smithi019.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-02-07T12:10:39.677 INFO:teuthology.orchestra.run.smithi019.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-02-07T12:10:39.677 INFO:teuthology.orchestra.run.smithi019.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-02-07T12:10:39.677 INFO:teuthology.orchestra.run.smithi019.stdout:Packager : builder@centos.org 2024-02-07T12:10:39.677 INFO:teuthology.orchestra.run.smithi019.stdout:Vendor : CentOS 2024-02-07T12:10:39.677 INFO:teuthology.orchestra.run.smithi019.stdout:URL : http://www.gnu.org/software/grub/ 2024-02-07T12:10:39.678 INFO:teuthology.orchestra.run.smithi019.stdout:Summary : Support tools for GRUB. 2024-02-07T12:10:39.678 INFO:teuthology.orchestra.run.smithi019.stdout:Description : 2024-02-07T12:10:39.678 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:10:39.678 INFO:teuthology.orchestra.run.smithi019.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-02-07T12:10:39.678 INFO:teuthology.orchestra.run.smithi019.stdout:customizable bootloader with modular architecture. It supports a rich 2024-02-07T12:10:39.678 INFO:teuthology.orchestra.run.smithi019.stdout:variety of kernel formats, file systems, computer architectures and 2024-02-07T12:10:39.678 INFO:teuthology.orchestra.run.smithi019.stdout:hardware devices. 2024-02-07T12:10:39.678 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:10:39.678 INFO:teuthology.orchestra.run.smithi019.stdout:This subpackage provides tools for support of all platforms. 2024-02-07T12:10:39.679 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-02-07T12:10:39.679 INFO:teuthology.task.kernel:Updating grub on smithi019 to boot 5.14.0-418.el9.x86_64 2024-02-07T12:10:39.679 DEBUG:teuthology.orchestra.run.smithi019:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-02-07T12:10:40.053 INFO:teuthology.orchestra.run.smithi172.stderr:Generating grub configuration file ... 2024-02-07T12:10:40.239 INFO:teuthology.orchestra.run.smithi019.stderr:Generating grub configuration file ... 2024-02-07T12:10:41.400 INFO:teuthology.orchestra.run.smithi172.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-02-07T12:10:41.428 INFO:teuthology.orchestra.run.smithi172.stderr:done 2024-02-07T12:10:41.430 DEBUG:teuthology.orchestra.run.smithi172:> mktemp 2024-02-07T12:10:41.445 INFO:teuthology.orchestra.run.smithi172.stdout:/tmp/tmp.m8PSFLyH1I 2024-02-07T12:10:41.445 DEBUG:teuthology.orchestra.run.smithi172:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.m8PSFLyH1I 2024-02-07T12:10:41.513 DEBUG:teuthology.orchestra.run.smithi172:> sudo chmod 0666 /tmp/tmp.m8PSFLyH1I 2024-02-07T12:10:41.603 INFO:teuthology.orchestra.run.smithi019.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-02-07T12:10:41.643 INFO:teuthology.orchestra.run.smithi019.stderr:done 2024-02-07T12:10:41.645 DEBUG:teuthology.orchestra.run.smithi019:> mktemp 2024-02-07T12:10:41.649 DEBUG:teuthology.orchestra.remote:smithi172:/tmp/tmp.m8PSFLyH1I is 6KB 2024-02-07T12:10:41.664 INFO:teuthology.orchestra.run.smithi019.stdout:/tmp/tmp.r3r44QfrxS 2024-02-07T12:10:41.664 DEBUG:teuthology.orchestra.run.smithi019:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.r3r44QfrxS 2024-02-07T12:10:41.698 DEBUG:teuthology.orchestra.run.smithi172:> rm -fr /tmp/tmp.m8PSFLyH1I 2024-02-07T12:10:41.713 DEBUG:teuthology.orchestra.run.smithi172:> sudo /bin/ls /boot/loader/entries || true 2024-02-07T12:10:41.737 DEBUG:teuthology.orchestra.run.smithi019:> sudo chmod 0666 /tmp/tmp.r3r44QfrxS 2024-02-07T12:10:41.777 INFO:teuthology.orchestra.run.smithi172.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-02-07T12:10:41.777 INFO:teuthology.orchestra.run.smithi172.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-02-07T12:10:41.777 INFO:teuthology.orchestra.run.smithi172.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-02-07T12:10:41.777 INFO:teuthology.orchestra.run.smithi172.stdout:15949e3c88704300882482eddfd7baae-5.14.0-418.el9.x86_64.conf 2024-02-07T12:10:41.779 DEBUG:teuthology.orchestra.run.smithi172:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-418.el9.x86_64 2024-02-07T12:10:41.879 DEBUG:teuthology.orchestra.remote:smithi019:/tmp/tmp.r3r44QfrxS is 6KB 2024-02-07T12:10:41.928 DEBUG:teuthology.orchestra.run.smithi019:> rm -fr /tmp/tmp.r3r44QfrxS 2024-02-07T12:10:41.929 DEBUG:teuthology.orchestra.run.smithi172:> sudo shutdown -r now 2024-02-07T12:10:41.942 DEBUG:teuthology.orchestra.run.smithi019:> sudo /bin/ls /boot/loader/entries || true 2024-02-07T12:10:42.020 INFO:teuthology.orchestra.run.smithi019.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-02-07T12:10:42.020 INFO:teuthology.orchestra.run.smithi019.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-02-07T12:10:42.020 INFO:teuthology.orchestra.run.smithi019.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-02-07T12:10:42.020 INFO:teuthology.orchestra.run.smithi019.stdout:15949e3c88704300882482eddfd7baae-5.14.0-418.el9.x86_64.conf 2024-02-07T12:10:42.022 DEBUG:teuthology.orchestra.run.smithi019:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-418.el9.x86_64 2024-02-07T12:10:42.182 DEBUG:teuthology.orchestra.run.smithi019:> sudo shutdown -r now 2024-02-07T12:11:11.959 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-02-07T12:11:11.959 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi172.front.sepia.ceph.com' 2024-02-07T12:11:11.961 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi172.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-07T12:11:12.186 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-02-07T12:11:12.186 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi019.front.sepia.ceph.com' 2024-02-07T12:11:12.186 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi019.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-07T12:11:30.496 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.172 2024-02-07T12:11:30.748 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.19 2024-02-07T12:11:39.504 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi172.front.sepia.ceph.com' 2024-02-07T12:11:39.505 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi172.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-07T12:11:39.749 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi019.front.sepia.ceph.com' 2024-02-07T12:11:39.750 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi019.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-07T12:11:42.556 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.172 2024-02-07T12:11:54.569 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi172.front.sepia.ceph.com' 2024-02-07T12:11:54.570 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi172.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-07T12:12:12.988 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.172 2024-02-07T12:12:27.999 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi172.front.sepia.ceph.com' 2024-02-07T12:12:28.000 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi172.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-07T12:12:28.447 DEBUG:teuthology.orchestra.run.smithi172:> true 2024-02-07T12:12:28.867 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi172.front.sepia.ceph.com' 2024-02-07T12:12:28.867 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-418.el9.x86_64"... 2024-02-07T12:12:28.868 DEBUG:teuthology.orchestra.run.smithi172:> uname -r 2024-02-07T12:12:28.882 INFO:teuthology.orchestra.run.smithi172.stdout:5.14.0-418.el9.x86_64 2024-02-07T12:12:28.882 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-418.el9.x86_64 vs 5.14.0-418.el9.x86_64 2024-02-07T12:12:28.882 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-02-07T12:12:28.882 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-02-07T12:12:29.883 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-02-07T12:12:29.883 DEBUG:teuthology.orchestra.run.smithi172:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-02-07T12:12:29.963 INFO:teuthology.orchestra.run.smithi172.stdout:ttyS1 2024-02-07T12:12:30.017 DEBUG:teuthology.parallel:result is None 2024-02-07T12:12:39.751 DEBUG:teuthology.orchestra.remote:timed out 2024-02-07T12:12:51.753 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi019.front.sepia.ceph.com' 2024-02-07T12:12:51.754 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi019.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-07T12:12:52.164 DEBUG:teuthology.orchestra.run.smithi019:> true 2024-02-07T12:12:52.555 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi019.front.sepia.ceph.com' 2024-02-07T12:12:52.556 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-418.el9.x86_64"... 2024-02-07T12:12:52.556 DEBUG:teuthology.orchestra.run.smithi019:> uname -r 2024-02-07T12:12:52.571 INFO:teuthology.orchestra.run.smithi019.stdout:5.14.0-418.el9.x86_64 2024-02-07T12:12:52.571 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-418.el9.x86_64 vs 5.14.0-418.el9.x86_64 2024-02-07T12:12:52.571 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-02-07T12:12:52.571 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-02-07T12:12:53.572 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-02-07T12:12:53.572 DEBUG:teuthology.orchestra.run.smithi019:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-02-07T12:12:53.630 INFO:teuthology.orchestra.run.smithi019.stdout:ttyS1 2024-02-07T12:12:53.666 DEBUG:teuthology.parallel:result is None 2024-02-07T12:12:53.666 INFO:teuthology.run_tasks:Running task internal.base... 2024-02-07T12:12:53.676 INFO:teuthology.task.internal:Creating test directory... 2024-02-07T12:12:53.676 DEBUG:teuthology.orchestra.run.smithi019:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-02-07T12:12:53.679 DEBUG:teuthology.orchestra.run.smithi172:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-02-07T12:12:53.698 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-02-07T12:12:53.705 INFO:teuthology.run_tasks:Running task internal.archive... 2024-02-07T12:12:53.711 INFO:teuthology.task.internal:Creating archive directory... 2024-02-07T12:12:53.711 DEBUG:teuthology.orchestra.run.smithi019:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-02-07T12:12:53.735 DEBUG:teuthology.orchestra.run.smithi172:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-02-07T12:12:53.769 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-02-07T12:12:53.776 INFO:teuthology.task.internal:Enabling coredump saving... 2024-02-07T12:12:53.776 DEBUG:teuthology.orchestra.run.smithi019:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-02-07T12:12:53.800 DEBUG:teuthology.orchestra.run.smithi172:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-02-07T12:12:53.834 INFO:teuthology.orchestra.run.smithi019.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-02-07T12:12:53.842 INFO:teuthology.orchestra.run.smithi172.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-02-07T12:12:53.848 INFO:teuthology.orchestra.run.smithi019.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-02-07T12:12:53.856 INFO:teuthology.orchestra.run.smithi172.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-02-07T12:12:53.858 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-02-07T12:12:53.865 INFO:teuthology.task.internal:Configuring sudo... 2024-02-07T12:12:53.865 DEBUG:teuthology.orchestra.run.smithi019:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-02-07T12:12:53.892 DEBUG:teuthology.orchestra.run.smithi172:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-02-07T12:12:53.927 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-02-07T12:12:53.937 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-02-07T12:12:53.937 DEBUG:teuthology.orchestra.run.smithi019:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-02-07T12:12:53.961 DEBUG:teuthology.orchestra.run.smithi172:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-02-07T12:12:53.983 DEBUG:teuthology.orchestra.run.smithi019:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-02-07T12:12:54.059 DEBUG:teuthology.orchestra.run.smithi019:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-02-07T12:12:54.106 DEBUG:teuthology.orchestra.run.smithi019:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-02-07T12:12:54.188 DEBUG:teuthology.orchestra.run.smithi019:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-02-07T12:12:54.230 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2024-02-07T12:12:54.230 DEBUG:teuthology.orchestra.run.smithi019:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-02-07T12:12:54.298 DEBUG:teuthology.orchestra.run.smithi172:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-02-07T12:12:54.341 DEBUG:teuthology.orchestra.run.smithi172:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-02-07T12:12:54.394 DEBUG:teuthology.orchestra.run.smithi172:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-02-07T12:12:54.478 DEBUG:teuthology.orchestra.run.smithi172:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-02-07T12:12:54.522 DEBUG:teuthology.orchestra.run.smithi172:> set -ex 2024-02-07T12:12:54.522 DEBUG:teuthology.orchestra.run.smithi172:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-02-07T12:12:54.589 DEBUG:teuthology.orchestra.run.smithi019:> sudo service rsyslog restart 2024-02-07T12:12:54.592 DEBUG:teuthology.orchestra.run.smithi172:> sudo service rsyslog restart 2024-02-07T12:12:54.633 INFO:teuthology.orchestra.run.smithi019.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-02-07T12:12:54.679 INFO:teuthology.orchestra.run.smithi172.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-02-07T12:12:55.154 INFO:teuthology.run_tasks:Running task internal.timer... 2024-02-07T12:12:55.162 INFO:teuthology.task.internal:Starting timer... 2024-02-07T12:12:55.163 INFO:teuthology.run_tasks:Running task pcp... 2024-02-07T12:12:55.174 INFO:teuthology.run_tasks:Running task selinux... 2024-02-07T12:12:55.183 DEBUG:teuthology.orchestra.run.smithi019:> sudo service auditd rotate 2024-02-07T12:12:55.245 INFO:teuthology.orchestra.run.smithi019.stdout:Rotating logs: 2024-02-07T12:12:55.247 DEBUG:teuthology.orchestra.run.smithi172:> sudo service auditd rotate 2024-02-07T12:12:55.294 INFO:teuthology.orchestra.run.smithi172.stdout:Rotating logs: 2024-02-07T12:12:55.296 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-02-07T12:12:55.296 DEBUG:teuthology.orchestra.run.smithi019:> /usr/sbin/getenforce 2024-02-07T12:12:55.317 INFO:teuthology.orchestra.run.smithi019.stdout:Permissive 2024-02-07T12:12:55.318 DEBUG:teuthology.orchestra.run.smithi172:> /usr/sbin/getenforce 2024-02-07T12:12:55.342 INFO:teuthology.orchestra.run.smithi172.stdout:Permissive 2024-02-07T12:12:55.343 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi019.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi172.front.sepia.ceph.com': 'permissive'} 2024-02-07T12:12:55.343 DEBUG:teuthology.orchestra.run.smithi019:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-02-07T12:12:55.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-02-07T12:12:55.387 DEBUG:teuthology.orchestra.run.smithi172:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-02-07T12:12:55.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-02-07T12:12:55.415 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-02-07T12:12:55.415 DEBUG:teuthology.orchestra.run.smithi019:> sudo /usr/sbin/setenforce permissive 2024-02-07T12:12:55.469 DEBUG:teuthology.orchestra.run.smithi172:> sudo /usr/sbin/setenforce permissive 2024-02-07T12:12:55.503 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-02-07T12:12:55.513 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-02-07T12:12:55.531 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-02-07T12:12:55.533 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi019.front.sepia.ceph.com,smithi172.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-02-07T12:17:05.313 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi019.front.sepia.ceph.com'), Remote(name='ubuntu@smithi172.front.sepia.ceph.com')] 2024-02-07T12:17:05.314 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi019.front.sepia.ceph.com' 2024-02-07T12:17:05.316 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi019.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-07T12:17:05.408 DEBUG:teuthology.orchestra.run.smithi019:> true 2024-02-07T12:17:05.487 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi019.front.sepia.ceph.com' 2024-02-07T12:17:05.487 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi172.front.sepia.ceph.com' 2024-02-07T12:17:05.488 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi172.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-07T12:17:05.580 DEBUG:teuthology.orchestra.run.smithi172:> true 2024-02-07T12:17:05.651 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi172.front.sepia.ceph.com' 2024-02-07T12:17:05.652 INFO:teuthology.run_tasks:Running task clock... 2024-02-07T12:17:05.663 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-02-07T12:17:05.663 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-02-07T12:17:05.664 DEBUG:teuthology.orchestra.run.smithi019:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-02-07T12:17:05.666 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-02-07T12:17:05.666 DEBUG:teuthology.orchestra.run.smithi172:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-02-07T12:17:05.696 INFO:teuthology.orchestra.run.smithi019.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-02-07T12:17:05.713 INFO:teuthology.orchestra.run.smithi019.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-02-07T12:17:05.723 INFO:teuthology.orchestra.run.smithi172.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-02-07T12:17:05.740 INFO:teuthology.orchestra.run.smithi172.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-02-07T12:17:05.743 INFO:teuthology.orchestra.run.smithi019.stderr:sudo: ntpd: command not found 2024-02-07T12:17:05.757 INFO:teuthology.orchestra.run.smithi019.stdout:506 Cannot talk to daemon 2024-02-07T12:17:05.771 INFO:teuthology.orchestra.run.smithi172.stderr:sudo: ntpd: command not found 2024-02-07T12:17:05.773 INFO:teuthology.orchestra.run.smithi019.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-02-07T12:17:05.784 INFO:teuthology.orchestra.run.smithi172.stdout:506 Cannot talk to daemon 2024-02-07T12:17:05.790 INFO:teuthology.orchestra.run.smithi019.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-02-07T12:17:05.801 INFO:teuthology.orchestra.run.smithi172.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-02-07T12:17:05.817 INFO:teuthology.orchestra.run.smithi172.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-02-07T12:17:05.846 INFO:teuthology.orchestra.run.smithi019.stderr:bash: line 1: ntpq: command not found 2024-02-07T12:17:05.849 INFO:teuthology.orchestra.run.smithi019.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-02-07T12:17:05.849 INFO:teuthology.orchestra.run.smithi019.stdout:=============================================================================== 2024-02-07T12:17:05.849 INFO:teuthology.orchestra.run.smithi019.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-02-07T12:17:05.849 INFO:teuthology.orchestra.run.smithi019.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-02-07T12:17:05.849 INFO:teuthology.orchestra.run.smithi019.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-02-07T12:17:05.849 INFO:teuthology.orchestra.run.smithi019.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-02-07T12:17:05.864 INFO:teuthology.orchestra.run.smithi172.stderr:bash: line 1: ntpq: command not found 2024-02-07T12:17:05.868 INFO:teuthology.orchestra.run.smithi172.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-02-07T12:17:05.868 INFO:teuthology.orchestra.run.smithi172.stdout:=============================================================================== 2024-02-07T12:17:05.868 INFO:teuthology.orchestra.run.smithi172.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-02-07T12:17:05.868 INFO:teuthology.orchestra.run.smithi172.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-02-07T12:17:05.868 INFO:teuthology.orchestra.run.smithi172.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-02-07T12:17:05.868 INFO:teuthology.orchestra.run.smithi172.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-02-07T12:17:05.869 INFO:teuthology.run_tasks:Running task cephadm... 2024-02-07T12:17:05.980 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1, 'mgr/cephadm/use_agent': False}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '44458ccedb18241cc2e17f10cdab3d1385399261'} 2024-02-07T12:17:05.980 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-02-07T12:17:05.981 INFO:tasks.cephadm:Cluster fsid is ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:17:05.981 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-02-07T12:17:05.981 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.19', 'mon.c': '[v2:172.21.15.19:3301,v1:172.21.15.19:6790]', 'mon.b': '172.21.15.172'} 2024-02-07T12:17:05.982 INFO:tasks.cephadm:First mon is mon.a on smithi019 2024-02-07T12:17:05.982 INFO:tasks.cephadm:First mgr is y 2024-02-07T12:17:05.982 INFO:tasks.cephadm:Normalizing hostnames... 2024-02-07T12:17:05.982 DEBUG:teuthology.orchestra.run.smithi019:> sudo hostname $(hostname -s) 2024-02-07T12:17:06.009 DEBUG:teuthology.orchestra.run.smithi172:> sudo hostname $(hostname -s) 2024-02-07T12:17:06.037 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-02-07T12:17:06.038 DEBUG:teuthology.orchestra.run.smithi019:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-02-07T12:17:06.319 INFO:teuthology.orchestra.run.smithi019.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Feb 7 12:17 /home/ubuntu/cephtest/cephadm 2024-02-07T12:17:06.319 DEBUG:teuthology.orchestra.run.smithi172:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-02-07T12:17:06.487 INFO:teuthology.orchestra.run.smithi172.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Feb 7 12:17 /home/ubuntu/cephtest/cephadm 2024-02-07T12:17:06.488 DEBUG:teuthology.orchestra.run.smithi019:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-02-07T12:17:06.503 DEBUG:teuthology.orchestra.run.smithi172:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-02-07T12:17:06.529 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-02-07T12:17:06.529 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-02-07T12:17:06.546 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-02-07T12:17:07.370 INFO:teuthology.orchestra.run.smithi019.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-02-07T12:17:07.377 INFO:teuthology.orchestra.run.smithi172.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-02-07T12:17:28.736 INFO:teuthology.orchestra.run.smithi172.stdout:{ 2024-02-07T12:17:28.736 INFO:teuthology.orchestra.run.smithi172.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-02-07T12:17:28.736 INFO:teuthology.orchestra.run.smithi172.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-02-07T12:17:28.737 INFO:teuthology.orchestra.run.smithi172.stdout: "repo_digests": [ 2024-02-07T12:17:28.737 INFO:teuthology.orchestra.run.smithi172.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-02-07T12:17:28.737 INFO:teuthology.orchestra.run.smithi172.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-02-07T12:17:28.737 INFO:teuthology.orchestra.run.smithi172.stdout: ] 2024-02-07T12:17:28.737 INFO:teuthology.orchestra.run.smithi172.stdout:} 2024-02-07T12:17:29.256 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:17:29.256 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-02-07T12:17:29.256 INFO:teuthology.orchestra.run.smithi019.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-02-07T12:17:29.256 INFO:teuthology.orchestra.run.smithi019.stdout: "repo_digests": [ 2024-02-07T12:17:29.256 INFO:teuthology.orchestra.run.smithi019.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-02-07T12:17:29.257 INFO:teuthology.orchestra.run.smithi019.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-02-07T12:17:29.257 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2024-02-07T12:17:29.257 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:17:29.275 DEBUG:teuthology.orchestra.run.smithi019:> sudo mkdir -p /etc/ceph 2024-02-07T12:17:29.322 DEBUG:teuthology.orchestra.run.smithi172:> sudo mkdir -p /etc/ceph 2024-02-07T12:17:29.354 DEBUG:teuthology.orchestra.run.smithi019:> sudo chmod 777 /etc/ceph 2024-02-07T12:17:29.381 DEBUG:teuthology.orchestra.run.smithi172:> sudo chmod 777 /etc/ceph 2024-02-07T12:17:29.422 INFO:tasks.cephadm:Writing seed config... 2024-02-07T12:17:29.423 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-02-07T12:17:29.424 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-02-07T12:17:29.424 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-02-07T12:17:29.424 INFO:tasks.cephadm: override: [mgr] mgr/cephadm/use_agent = False 2024-02-07T12:17:29.424 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-02-07T12:17:29.424 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-02-07T12:17:29.424 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-02-07T12:17:29.424 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-02-07T12:17:29.424 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-02-07T12:17:29.425 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2024-02-07T12:17:29.425 DEBUG:teuthology.orchestra.run.smithi019:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-02-07T12:17:29.440 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = ced2783c-c5b2-11ee-95b6-87774f69a715 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = true bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 mgr/cephadm/use_agent = False [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-02-07T12:17:29.440 DEBUG:teuthology.orchestra.run.smithi019:mon.a> sudo journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mon.a.service 2024-02-07T12:17:29.486 DEBUG:teuthology.orchestra.run.smithi019:mgr.y> sudo journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mgr.y.service 2024-02-07T12:17:29.530 INFO:tasks.cephadm:Bootstrapping... 2024-02-07T12:17:29.530 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.19 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-02-07T12:17:29.731 INFO:teuthology.orchestra.run.smithi019.stderr:-------------------------------------------------------------------------------- 2024-02-07T12:17:29.731 INFO:teuthology.orchestra.run.smithi019.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', 'ced2783c-c5b2-11ee-95b6-87774f69a715', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.19', '--skip-admin-label'] 2024-02-07T12:17:29.754 INFO:teuthology.orchestra.run.smithi019.stderr:/bin/podman: 4.9.3-dev 2024-02-07T12:17:29.756 INFO:teuthology.orchestra.run.smithi019.stderr:Verifying podman|docker is present... 2024-02-07T12:17:29.777 INFO:teuthology.orchestra.run.smithi019.stderr:/bin/podman: 4.9.3-dev 2024-02-07T12:17:29.779 INFO:teuthology.orchestra.run.smithi019.stderr:Verifying lvm2 is present... 2024-02-07T12:17:29.779 INFO:teuthology.orchestra.run.smithi019.stderr:Verifying time synchronization is in place... 2024-02-07T12:17:29.786 INFO:teuthology.orchestra.run.smithi019.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-02-07T12:17:29.792 INFO:teuthology.orchestra.run.smithi019.stderr:systemctl: inactive 2024-02-07T12:17:29.799 INFO:teuthology.orchestra.run.smithi019.stderr:systemctl: enabled 2024-02-07T12:17:29.804 INFO:teuthology.orchestra.run.smithi019.stderr:systemctl: active 2024-02-07T12:17:29.805 INFO:teuthology.orchestra.run.smithi019.stderr:Unit chronyd.service is enabled and running 2024-02-07T12:17:29.805 INFO:teuthology.orchestra.run.smithi019.stderr:Repeating the final host check... 2024-02-07T12:17:29.825 INFO:teuthology.orchestra.run.smithi019.stderr:/bin/podman: 4.9.3-dev 2024-02-07T12:17:29.827 INFO:teuthology.orchestra.run.smithi019.stderr:podman (/bin/podman) version 4.9.3 is present 2024-02-07T12:17:29.827 INFO:teuthology.orchestra.run.smithi019.stderr:systemctl is present 2024-02-07T12:17:29.827 INFO:teuthology.orchestra.run.smithi019.stderr:lvcreate is present 2024-02-07T12:17:29.833 INFO:teuthology.orchestra.run.smithi019.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-02-07T12:17:29.839 INFO:teuthology.orchestra.run.smithi019.stderr:systemctl: inactive 2024-02-07T12:17:29.845 INFO:teuthology.orchestra.run.smithi019.stderr:systemctl: enabled 2024-02-07T12:17:29.851 INFO:teuthology.orchestra.run.smithi019.stderr:systemctl: active 2024-02-07T12:17:29.852 INFO:teuthology.orchestra.run.smithi019.stderr:Unit chronyd.service is enabled and running 2024-02-07T12:17:29.852 INFO:teuthology.orchestra.run.smithi019.stderr:Host looks OK 2024-02-07T12:17:29.852 INFO:teuthology.orchestra.run.smithi019.stderr:Cluster fsid: ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:17:29.852 INFO:teuthology.orchestra.run.smithi019.stderr:Acquiring lock 140192864804768 on /run/cephadm/ced2783c-c5b2-11ee-95b6-87774f69a715.lock 2024-02-07T12:17:29.853 INFO:teuthology.orchestra.run.smithi019.stderr:Lock 140192864804768 acquired on /run/cephadm/ced2783c-c5b2-11ee-95b6-87774f69a715.lock 2024-02-07T12:17:29.853 INFO:teuthology.orchestra.run.smithi019.stderr:Verifying IP 172.21.15.19 port 3300 ... 2024-02-07T12:17:29.853 INFO:teuthology.orchestra.run.smithi019.stderr:Verifying IP 172.21.15.19 port 6789 ... 2024-02-07T12:17:29.853 INFO:teuthology.orchestra.run.smithi019.stderr:Base mon IP is 172.21.15.19, final addrv is [v2:172.21.15.19:3300,v1:172.21.15.19:6789] 2024-02-07T12:17:29.855 INFO:teuthology.orchestra.run.smithi019.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.19 metric 100 2024-02-07T12:17:29.855 INFO:teuthology.orchestra.run.smithi019.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.19 metric 100 2024-02-07T12:17:29.858 INFO:teuthology.orchestra.run.smithi019.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-02-07T12:17:29.858 INFO:teuthology.orchestra.run.smithi019.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 1024 pref medium 2024-02-07T12:17:29.858 INFO:teuthology.orchestra.run.smithi019.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 100 pref medium 2024-02-07T12:17:29.860 INFO:teuthology.orchestra.run.smithi019.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-02-07T12:17:29.860 INFO:teuthology.orchestra.run.smithi019.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-02-07T12:17:29.861 INFO:teuthology.orchestra.run.smithi019.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-02-07T12:17:29.861 INFO:teuthology.orchestra.run.smithi019.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2024-02-07T12:17:29.861 INFO:teuthology.orchestra.run.smithi019.stderr:/sbin/ip: inet6 fe80::ec4:7aff:febd:161e/64 scope link noprefixroute 2024-02-07T12:17:29.861 INFO:teuthology.orchestra.run.smithi019.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-02-07T12:17:29.861 INFO:teuthology.orchestra.run.smithi019.stderr:Mon IP `172.21.15.19` is in CIDR network `172.21.0.0/20` 2024-02-07T12:17:29.861 INFO:teuthology.orchestra.run.smithi019.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-02-07T12:17:29.862 INFO:teuthology.orchestra.run.smithi019.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-02-07T12:17:29.965 INFO:teuthology.orchestra.run.smithi019.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-02-07T12:17:30.295 INFO:teuthology.orchestra.run.smithi019.stderr:/bin/podman: Getting image source signatures 2024-02-07T12:17:30.295 INFO:teuthology.orchestra.run.smithi019.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-02-07T12:17:30.295 INFO:teuthology.orchestra.run.smithi019.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-02-07T12:17:30.295 INFO:teuthology.orchestra.run.smithi019.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-02-07T12:17:30.295 INFO:teuthology.orchestra.run.smithi019.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-02-07T12:17:30.295 INFO:teuthology.orchestra.run.smithi019.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-02-07T12:17:30.296 INFO:teuthology.orchestra.run.smithi019.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-02-07T12:17:30.460 INFO:teuthology.orchestra.run.smithi019.stderr:/bin/podman: Writing manifest to image destination 2024-02-07T12:17:30.596 INFO:teuthology.orchestra.run.smithi019.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-02-07T12:17:31.027 INFO:teuthology.orchestra.run.smithi019.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-02-07T12:17:31.487 INFO:teuthology.orchestra.run.smithi019.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-02-07T12:17:31.487 INFO:teuthology.orchestra.run.smithi019.stderr:Extracting ceph user uid/gid from container image... 2024-02-07T12:17:31.845 INFO:teuthology.orchestra.run.smithi019.stderr:stat: 167 167 2024-02-07T12:17:32.351 INFO:teuthology.orchestra.run.smithi019.stderr:Creating initial keys... 2024-02-07T12:17:32.784 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph-authtool: AQDcdMNlwqqvKRAAdUn+y3CHL2u89Db/uyp5eg== 2024-02-07T12:17:33.772 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph-authtool: AQDddMNlXijnKhAAUPT2XStMt7fv8fStM9LTzQ== 2024-02-07T12:17:34.661 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph-authtool: AQDedMNl8e5BIhAArjV9wnNJavGbmjVn46nfaA== 2024-02-07T12:17:35.160 INFO:teuthology.orchestra.run.smithi019.stderr:Creating initial monmap... 2024-02-07T12:17:35.592 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-02-07T12:17:35.592 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-02-07T12:17:35.593 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:17:35.593 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-02-07T12:17:36.106 INFO:teuthology.orchestra.run.smithi019.stderr:monmaptool for a [v2:172.21.15.19:3300,v1:172.21.15.19:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-02-07T12:17:36.106 INFO:teuthology.orchestra.run.smithi019.stderr:setting min_mon_release = octopus 2024-02-07T12:17:36.107 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/monmaptool: set fsid to ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:17:36.107 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-02-07T12:17:36.107 INFO:teuthology.orchestra.run.smithi019.stderr: 2024-02-07T12:17:36.107 INFO:teuthology.orchestra.run.smithi019.stderr:Creating mon... 2024-02-07T12:17:36.954 INFO:teuthology.orchestra.run.smithi019.stderr:create mon.a on 2024-02-07T12:17:37.163 INFO:teuthology.orchestra.run.smithi019.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-02-07T12:17:37.311 INFO:teuthology.orchestra.run.smithi019.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-ced2783c-c5b2-11ee-95b6-87774f69a715.target → /etc/systemd/system/ceph-ced2783c-c5b2-11ee-95b6-87774f69a715.target. 2024-02-07T12:17:37.311 INFO:teuthology.orchestra.run.smithi019.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-ced2783c-c5b2-11ee-95b6-87774f69a715.target → /etc/systemd/system/ceph-ced2783c-c5b2-11ee-95b6-87774f69a715.target. 2024-02-07T12:17:37.601 INFO:teuthology.orchestra.run.smithi019.stderr:systemctl: Failed to reset failed state of unit ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mon.a.service: Unit ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mon.a.service not loaded. 2024-02-07T12:17:37.608 INFO:teuthology.orchestra.run.smithi019.stderr:systemctl: Created symlink /etc/systemd/system/ceph-ced2783c-c5b2-11ee-95b6-87774f69a715.target.wants/ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mon.a.service → /etc/systemd/system/ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@.service. 2024-02-07T12:17:38.255 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:38 smithi019 podman[24209]: 2024-02-07 12:17:38.189569821 +0000 UTC m=+0.311749566 container start 7922aa013b188ef8e2d7063c3bce36a5af4c19e07bddea6d3690edb20ca37414 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-a, com.redhat.component=centos-stream-container, release=754, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, vcs-type=git, maintainer=Guillaume Abrioux , name=centos-stream, GIT_BRANCH=HEAD, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-02-07T12:17:38.255 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:38 smithi019 bash[24209]: 7922aa013b188ef8e2d7063c3bce36a5af4c19e07bddea6d3690edb20ca37414 2024-02-07T12:17:38.267 INFO:teuthology.orchestra.run.smithi019.stderr:systemctl: disabled 2024-02-07T12:17:38.273 INFO:teuthology.orchestra.run.smithi019.stderr:systemctl: inactive 2024-02-07T12:17:38.274 INFO:teuthology.orchestra.run.smithi019.stderr:firewalld.service is not enabled 2024-02-07T12:17:38.274 INFO:teuthology.orchestra.run.smithi019.stderr:Not possible to enable service . firewalld.service is not available 2024-02-07T12:17:38.275 INFO:teuthology.orchestra.run.smithi019.stderr:Waiting for mon to start... 2024-02-07T12:17:38.275 INFO:teuthology.orchestra.run.smithi019.stderr:Waiting for mon... 2024-02-07T12:17:38.506 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:38 smithi019 ceph-mon[24228]: mkfs ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:17:38.506 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:38 smithi019 systemd[1]: Started Ceph mon.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:17:38.507 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:38 smithi019 ceph-mon[24228]: mon.a is new leader, mons a in quorum (ranks 0) 2024-02-07T12:17:38.801 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: cluster: 2024-02-07T12:17:38.801 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: id: ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:17:38.802 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-02-07T12:17:38.802 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: 2024-02-07T12:17:38.802 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: services: 2024-02-07T12:17:38.802 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.547749s) 2024-02-07T12:17:38.802 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: mgr: no daemons active 2024-02-07T12:17:38.802 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-02-07T12:17:38.802 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: 2024-02-07T12:17:38.802 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: data: 2024-02-07T12:17:38.802 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-02-07T12:17:38.802 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-02-07T12:17:38.802 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-02-07T12:17:38.802 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: pgs: 2024-02-07T12:17:38.802 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: 2024-02-07T12:17:39.372 INFO:teuthology.orchestra.run.smithi019.stderr:mon is available 2024-02-07T12:17:39.372 INFO:teuthology.orchestra.run.smithi019.stderr:Assimilating anything we can from ceph.conf... 2024-02-07T12:17:39.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:39 smithi019 ceph-mon[24228]: mon.a is new leader, mons a in quorum (ranks 0) 2024-02-07T12:17:39.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:39 smithi019 ceph-mon[24228]: monmap e1: 1 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0]} 2024-02-07T12:17:39.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:39 smithi019 ceph-mon[24228]: fsmap 2024-02-07T12:17:39.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:39 smithi019 ceph-mon[24228]: osdmap e1: 0 total, 0 up, 0 in 2024-02-07T12:17:39.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:39 smithi019 ceph-mon[24228]: mgrmap e1: no daemons active 2024-02-07T12:17:39.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:39 smithi019 ceph-mon[24228]: from='client.? 172.21.15.19:0/3331551963' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-02-07T12:17:39.912 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: 2024-02-07T12:17:39.912 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: [global] 2024-02-07T12:17:39.913 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: fsid = ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:17:39.913 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.19:3300,v1:172.21.15.19:6789] 2024-02-07T12:17:39.913 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-02-07T12:17:39.913 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-02-07T12:17:39.913 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-02-07T12:17:39.913 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-02-07T12:17:39.913 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: 2024-02-07T12:17:39.913 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: [mgr] 2024-02-07T12:17:39.913 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: mgr/cephadm/use_agent = False 2024-02-07T12:17:39.913 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-02-07T12:17:39.913 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: 2024-02-07T12:17:39.913 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: [osd] 2024-02-07T12:17:39.913 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-02-07T12:17:39.914 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-02-07T12:17:40.419 INFO:teuthology.orchestra.run.smithi019.stderr:Generating new minimal ceph.conf... 2024-02-07T12:17:40.542 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:40 smithi019 ceph-mon[24228]: from='client.? 172.21.15.19:0/1157944615' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-02-07T12:17:40.542 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:40 smithi019 ceph-mon[24228]: from='client.? 172.21.15.19:0/1157944615' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-02-07T12:17:41.467 INFO:teuthology.orchestra.run.smithi019.stderr:Restarting the monitor... 2024-02-07T12:17:41.513 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:41 smithi019 ceph-mon[24228]: from='client.? 172.21.15.19:0/2861719752' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:17:41.513 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:41 smithi019 systemd[1]: Stopping Ceph mon.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:17:41.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:41 smithi019 bash[24446]: Error: no container with name or ID "ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon.a" found: no such container 2024-02-07T12:17:41.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-a[24224]: 2024-02-07T12:17:41.606+0000 7f3a1daea700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:17:41.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-a[24224]: 2024-02-07T12:17:41.606+0000 7f3a1daea700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-02-07T12:17:41.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:41 smithi019 podman[24459]: 2024-02-07 12:17:41.638003229 +0000 UTC m=+0.117801275 container died 7922aa013b188ef8e2d7063c3bce36a5af4c19e07bddea6d3690edb20ca37414 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-a, vcs-type=git, io.openshift.expose-services=, release=754, version=8, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True) 2024-02-07T12:17:42.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:41 smithi019 podman[24459]: 2024-02-07 12:17:41.97345913 +0000 UTC m=+0.453257185 container cleanup 7922aa013b188ef8e2d7063c3bce36a5af4c19e07bddea6d3690edb20ca37414 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-a, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, io.openshift.expose-services=, distribution-scope=public, maintainer=Guillaume Abrioux , name=centos-stream, RELEASE=HEAD, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, release=754, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-02-07T12:17:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:41 smithi019 bash[24459]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-a 2024-02-07T12:17:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:41 smithi019 bash[24485]: Error: no container with name or ID "ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon.a" found: no such container 2024-02-07T12:17:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 podman[24473]: 2024-02-07 12:17:42.141101543 +0000 UTC m=+0.494637917 container remove 7922aa013b188ef8e2d7063c3bce36a5af4c19e07bddea6d3690edb20ca37414 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-a, build-date=2022-05-03T08:36:31.336870, vcs-type=git, vendor=Red Hat, Inc., io.openshift.expose-services=, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, distribution-scope=public, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, version=8, io.k8s.display-name=CentOS Stream 8, name=centos-stream) 2024-02-07T12:17:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mon.a.service: Deactivated successfully. 2024-02-07T12:17:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 systemd[1]: Stopped Ceph mon.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:17:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 systemd[1]: Starting Ceph mon.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:17:42.654 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 podman[24555]: 2024-02-07 12:17:42.366733249 +0000 UTC m=+0.114145168 container create f3a8a050cf1d560b64ce22cb4e363436d142dcdb71fe1e93d919b7adc4d6fcd8 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-a, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, io.openshift.expose-services=, vendor=Red Hat, Inc., architecture=x86_64, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, vcs-type=git, io.buildah.version=1.19.8, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, com.redhat.component=centos-stream-container, GIT_CLEAN=True) 2024-02-07T12:17:42.654 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 podman[24555]: 2024-02-07 12:17:42.270619531 +0000 UTC m=+0.018031458 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-02-07T12:17:42.654 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 podman[24555]: 2024-02-07 12:17:42.577578927 +0000 UTC m=+0.324990845 container init f3a8a050cf1d560b64ce22cb4e363436d142dcdb71fe1e93d919b7adc4d6fcd8 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-a, io.openshift.expose-services=, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, vcs-type=git, vendor=Red Hat, Inc., architecture=x86_64, com.redhat.component=centos-stream-container, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-02-07T12:17:42.655 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 podman[24555]: 2024-02-07 12:17:42.58071665 +0000 UTC m=+0.328128570 container start f3a8a050cf1d560b64ce22cb4e363436d142dcdb71fe1e93d919b7adc4d6fcd8 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-a, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, version=8, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, distribution-scope=public, com.redhat.component=centos-stream-container, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , name=centos-stream) 2024-02-07T12:17:42.655 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: set uid:gid to 167:167 (ceph:ceph) 2024-02-07T12:17:42.655 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-02-07T12:17:42.655 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: pidfile_write: ignore empty --pid-file 2024-02-07T12:17:42.655 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: load: jerasure load: lrc 2024-02-07T12:17:42.655 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: RocksDB version: 6.15.5 2024-02-07T12:17:42.655 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-02-07T12:17:42.655 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Compile date Apr 18 2022 2024-02-07T12:17:42.655 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: DB SUMMARY 2024-02-07T12:17:42.655 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: DB Session ID: KC93OT9YCA5NBNQI43LG 2024-02-07T12:17:42.655 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: CURRENT file: CURRENT 2024-02-07T12:17:42.655 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: IDENTITY file: IDENTITY 2024-02-07T12:17:42.656 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-02-07T12:17:42.656 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-02-07T12:17:42.656 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 82635 ; 2024-02-07T12:17:42.656 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.error_if_exists: 0 2024-02-07T12:17:42.656 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.create_if_missing: 0 2024-02-07T12:17:42.656 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.paranoid_checks: 1 2024-02-07T12:17:42.656 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-02-07T12:17:42.656 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.env: 0x5599c36c6860 2024-02-07T12:17:42.656 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.fs: Posix File System 2024-02-07T12:17:42.656 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.info_log: 0x5599c440fdc0 2024-02-07T12:17:42.656 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.max_file_opening_threads: 16 2024-02-07T12:17:42.656 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.statistics: (nil) 2024-02-07T12:17:42.657 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.use_fsync: 0 2024-02-07T12:17:42.657 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.max_log_file_size: 0 2024-02-07T12:17:42.657 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-02-07T12:17:42.658 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.log_file_time_to_roll: 0 2024-02-07T12:17:42.658 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.keep_log_file_num: 1000 2024-02-07T12:17:42.658 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.recycle_log_file_num: 0 2024-02-07T12:17:42.658 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.allow_fallocate: 1 2024-02-07T12:17:42.658 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.allow_mmap_reads: 0 2024-02-07T12:17:42.658 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.allow_mmap_writes: 0 2024-02-07T12:17:42.659 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.use_direct_reads: 0 2024-02-07T12:17:42.659 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-02-07T12:17:42.659 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.create_missing_column_families: 0 2024-02-07T12:17:42.659 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.db_log_dir: 2024-02-07T12:17:42.659 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-02-07T12:17:42.659 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.table_cache_numshardbits: 6 2024-02-07T12:17:42.659 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.WAL_ttl_seconds: 0 2024-02-07T12:17:42.659 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.WAL_size_limit_MB: 0 2024-02-07T12:17:42.659 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-02-07T12:17:42.659 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-02-07T12:17:42.659 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.is_fd_close_on_exec: 1 2024-02-07T12:17:42.659 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.advise_random_on_open: 1 2024-02-07T12:17:42.659 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.db_write_buffer_size: 0 2024-02-07T12:17:42.659 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.write_buffer_manager: 0x5599c45002a0 2024-02-07T12:17:42.659 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-02-07T12:17:42.660 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-02-07T12:17:42.660 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-02-07T12:17:42.660 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.use_adaptive_mutex: 0 2024-02-07T12:17:42.660 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.rate_limiter: (nil) 2024-02-07T12:17:42.660 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-02-07T12:17:42.660 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.wal_recovery_mode: 2 2024-02-07T12:17:42.660 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.enable_thread_tracking: 0 2024-02-07T12:17:42.660 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.enable_pipelined_write: 0 2024-02-07T12:17:42.660 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.unordered_write: 0 2024-02-07T12:17:42.660 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-02-07T12:17:42.660 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-02-07T12:17:42.663 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-02-07T12:17:42.663 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-02-07T12:17:42.663 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.row_cache: None 2024-02-07T12:17:42.663 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.wal_filter: None 2024-02-07T12:17:42.663 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-02-07T12:17:42.663 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.allow_ingest_behind: 0 2024-02-07T12:17:42.664 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.preserve_deletes: 0 2024-02-07T12:17:42.664 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.two_write_queues: 0 2024-02-07T12:17:42.664 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.manual_wal_flush: 0 2024-02-07T12:17:42.664 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.atomic_flush: 0 2024-02-07T12:17:42.664 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-02-07T12:17:42.664 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.persist_stats_to_disk: 0 2024-02-07T12:17:42.664 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.write_dbid_to_manifest: 0 2024-02-07T12:17:42.664 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.log_readahead_size: 0 2024-02-07T12:17:42.664 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-02-07T12:17:42.664 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.best_efforts_recovery: 0 2024-02-07T12:17:42.664 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-02-07T12:17:42.664 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-02-07T12:17:42.664 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.allow_data_in_errors: 0 2024-02-07T12:17:42.664 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.db_host_id: __hostname__ 2024-02-07T12:17:42.664 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.max_background_jobs: 2 2024-02-07T12:17:42.665 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.max_background_compactions: -1 2024-02-07T12:17:42.665 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.max_subcompactions: 1 2024-02-07T12:17:42.666 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-02-07T12:17:42.666 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-02-07T12:17:42.666 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.delayed_write_rate : 16777216 2024-02-07T12:17:42.666 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.max_total_wal_size: 0 2024-02-07T12:17:42.666 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-02-07T12:17:42.666 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.stats_dump_period_sec: 600 2024-02-07T12:17:42.666 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.stats_persist_period_sec: 600 2024-02-07T12:17:42.666 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-02-07T12:17:42.666 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.max_open_files: -1 2024-02-07T12:17:42.666 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.bytes_per_sync: 0 2024-02-07T12:17:42.666 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.wal_bytes_per_sync: 0 2024-02-07T12:17:42.666 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.strict_bytes_per_sync: 0 2024-02-07T12:17:42.666 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.compaction_readahead_size: 0 2024-02-07T12:17:42.666 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.max_background_flushes: -1 2024-02-07T12:17:42.667 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Compression algorithms supported: 2024-02-07T12:17:42.667 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-02-07T12:17:42.667 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: kZSTD supported: 0 2024-02-07T12:17:42.667 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: kXpressCompression supported: 0 2024-02-07T12:17:42.667 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: kLZ4HCCompression supported: 1 2024-02-07T12:17:42.668 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: kLZ4Compression supported: 1 2024-02-07T12:17:42.668 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: kBZip2Compression supported: 0 2024-02-07T12:17:42.668 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: kZlibCompression supported: 1 2024-02-07T12:17:42.668 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: kSnappyCompression supported: 1 2024-02-07T12:17:42.668 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Fast CRC32 supported: Supported on x86 2024-02-07T12:17:42.668 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-02-07T12:17:42.668 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-02-07T12:17:42.668 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-02-07T12:17:42.668 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.merge_operator: 2024-02-07T12:17:42.668 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.compaction_filter: None 2024-02-07T12:17:42.668 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.compaction_filter_factory: None 2024-02-07T12:17:42.669 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.sst_partitioner_factory: None 2024-02-07T12:17:42.669 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.memtable_factory: SkipListFactory 2024-02-07T12:17:42.669 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.table_factory: BlockBasedTable 2024-02-07T12:17:42.669 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5599c43dbd00) 2024-02-07T12:17:42.669 INFO:journalctl@ceph.mon.a.smithi019.stdout: cache_index_and_filter_blocks: 1 2024-02-07T12:17:42.669 INFO:journalctl@ceph.mon.a.smithi019.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-02-07T12:17:42.669 INFO:journalctl@ceph.mon.a.smithi019.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-02-07T12:17:42.669 INFO:journalctl@ceph.mon.a.smithi019.stdout: pin_top_level_index_and_filter: 1 2024-02-07T12:17:42.669 INFO:journalctl@ceph.mon.a.smithi019.stdout: index_type: 0 2024-02-07T12:17:42.669 INFO:journalctl@ceph.mon.a.smithi019.stdout: data_block_index_type: 0 2024-02-07T12:17:42.669 INFO:journalctl@ceph.mon.a.smithi019.stdout: index_shortening: 1 2024-02-07T12:17:42.669 INFO:journalctl@ceph.mon.a.smithi019.stdout: data_block_hash_table_util_ratio: 0.750000 2024-02-07T12:17:42.669 INFO:journalctl@ceph.mon.a.smithi019.stdout: hash_index_allow_collision: 1 2024-02-07T12:17:42.669 INFO:journalctl@ceph.mon.a.smithi019.stdout: checksum: 1 2024-02-07T12:17:42.670 INFO:journalctl@ceph.mon.a.smithi019.stdout: no_block_cache: 0 2024-02-07T12:17:42.670 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_cache: 0x5599c4446170 2024-02-07T12:17:42.670 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_cache_name: BinnedLRUCache 2024-02-07T12:17:42.670 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_cache_options: 2024-02-07T12:17:42.670 INFO:journalctl@ceph.mon.a.smithi019.stdout: capacity : 536870912 2024-02-07T12:17:42.670 INFO:journalctl@ceph.mon.a.smithi019.stdout: num_shard_bits : 4 2024-02-07T12:17:42.670 INFO:journalctl@ceph.mon.a.smithi019.stdout: strict_capacity_limit : 0 2024-02-07T12:17:42.670 INFO:journalctl@ceph.mon.a.smithi019.stdout: high_pri_pool_ratio: 0.000 2024-02-07T12:17:42.670 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_cache_compressed: (nil) 2024-02-07T12:17:42.670 INFO:journalctl@ceph.mon.a.smithi019.stdout: persistent_cache: (nil) 2024-02-07T12:17:42.670 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_size: 4096 2024-02-07T12:17:42.670 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_size_deviation: 10 2024-02-07T12:17:42.670 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_restart_interval: 16 2024-02-07T12:17:42.670 INFO:journalctl@ceph.mon.a.smithi019.stdout: index_block_restart_interval: 1 2024-02-07T12:17:42.671 INFO:journalctl@ceph.mon.a.smithi019.stdout: metadata_block_size: 4096 2024-02-07T12:17:42.671 INFO:journalctl@ceph.mon.a.smithi019.stdout: partition_filters: 0 2024-02-07T12:17:42.671 INFO:journalctl@ceph.mon.a.smithi019.stdout: use_delta_encoding: 1 2024-02-07T12:17:42.671 INFO:journalctl@ceph.mon.a.smithi019.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-02-07T12:17:42.671 INFO:journalctl@ceph.mon.a.smithi019.stdout: whole_key_filtering: 1 2024-02-07T12:17:42.671 INFO:journalctl@ceph.mon.a.smithi019.stdout: verify_compression: 0 2024-02-07T12:17:42.671 INFO:journalctl@ceph.mon.a.smithi019.stdout: read_amp_bytes_per_bit: 0 2024-02-07T12:17:42.671 INFO:journalctl@ceph.mon.a.smithi019.stdout: format_version: 4 2024-02-07T12:17:42.671 INFO:journalctl@ceph.mon.a.smithi019.stdout: enable_index_compression: 1 2024-02-07T12:17:42.671 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_align: 0 2024-02-07T12:17:42.671 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.write_buffer_size: 33554432 2024-02-07T12:17:42.671 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.max_write_buffer_number: 2 2024-02-07T12:17:42.671 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.compression: NoCompression 2024-02-07T12:17:42.671 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.bottommost_compression: Disabled 2024-02-07T12:17:42.671 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.prefix_extractor: nullptr 2024-02-07T12:17:42.671 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-02-07T12:17:42.672 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.num_levels: 7 2024-02-07T12:17:42.672 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-02-07T12:17:42.672 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-02-07T12:17:42.672 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-02-07T12:17:42.672 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-02-07T12:17:42.672 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-02-07T12:17:42.672 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-02-07T12:17:42.672 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-02-07T12:17:42.672 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-02-07T12:17:42.672 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-02-07T12:17:42.672 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-02-07T12:17:42.672 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.compression_opts.window_bits: -14 2024-02-07T12:17:42.672 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.compression_opts.level: 32767 2024-02-07T12:17:42.673 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.compression_opts.strategy: 0 2024-02-07T12:17:42.673 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-02-07T12:17:42.673 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-02-07T12:17:42.673 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-02-07T12:17:42.673 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.compression_opts.enabled: false 2024-02-07T12:17:42.673 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-02-07T12:17:42.673 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-02-07T12:17:42.673 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-02-07T12:17:42.673 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.target_file_size_base: 67108864 2024-02-07T12:17:42.674 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.target_file_size_multiplier: 1 2024-02-07T12:17:42.674 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-02-07T12:17:42.674 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-02-07T12:17:42.674 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-02-07T12:17:42.674 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-02-07T12:17:42.674 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-02-07T12:17:42.674 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-02-07T12:17:42.674 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-02-07T12:17:42.674 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-02-07T12:17:42.674 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-02-07T12:17:42.674 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-02-07T12:17:42.674 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-02-07T12:17:42.674 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-02-07T12:17:42.675 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.arena_block_size: 4194304 2024-02-07T12:17:42.675 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-02-07T12:17:42.675 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-02-07T12:17:42.675 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-02-07T12:17:42.675 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.disable_auto_compactions: 0 2024-02-07T12:17:42.675 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-02-07T12:17:42.675 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-02-07T12:17:42.675 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-02-07T12:17:42.675 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-02-07T12:17:42.675 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-02-07T12:17:42.675 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-02-07T12:17:42.675 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-02-07T12:17:42.675 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-02-07T12:17:42.675 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-02-07T12:17:42.676 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-02-07T12:17:42.676 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.table_properties_collectors: 2024-02-07T12:17:42.676 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.inplace_update_support: 0 2024-02-07T12:17:42.676 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.inplace_update_num_locks: 10000 2024-02-07T12:17:42.676 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-02-07T12:17:42.676 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-02-07T12:17:42.676 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.memtable_huge_page_size: 0 2024-02-07T12:17:42.676 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.bloom_locality: 0 2024-02-07T12:17:42.676 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.max_successive_merges: 0 2024-02-07T12:17:42.676 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.optimize_filters_for_hits: 0 2024-02-07T12:17:42.676 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.paranoid_file_checks: 0 2024-02-07T12:17:42.676 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.force_consistency_checks: 1 2024-02-07T12:17:42.676 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.report_bg_io_stats: 0 2024-02-07T12:17:42.676 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.ttl: 2592000 2024-02-07T12:17:42.677 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.periodic_compaction_seconds: 0 2024-02-07T12:17:42.677 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.enable_blob_files: false 2024-02-07T12:17:42.677 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.min_blob_size: 0 2024-02-07T12:17:42.677 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.blob_file_size: 268435456 2024-02-07T12:17:42.677 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.blob_compression_type: NoCompression 2024-02-07T12:17:42.677 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.enable_blob_garbage_collection: false 2024-02-07T12:17:42.677 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-02-07T12:17:42.677 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-02-07T12:17:42.677 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-02-07T12:17:42.677 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-02-07T12:17:42.677 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707308262606270, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-02-07T12:17:42.677 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-02-07T12:17:42.677 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-02-07T12:17:42.677 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707308262606966, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79185, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77448, "index_size": 225, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10027, "raw_average_key_size": 47, "raw_value_size": 71757, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 211, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1707308262, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "460f5071-39a7-44b5-b635-e5923fb6e0d9", "db_session_id": "KC93OT9YCA5NBNQI43LG"}} 2024-02-07T12:17:42.678 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-02-07T12:17:42.678 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707308262607291, "job": 1, "event": "recovery_finished"} 2024-02-07T12:17:42.678 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-02-07T12:17:42.678 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x5599c442ca80 2024-02-07T12:17:42.678 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: DB pointer 0x5599c443c000 2024-02-07T12:17:42.678 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-02-07T12:17:42.678 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-02-07T12:17:42.678 INFO:journalctl@ceph.mon.a.smithi019.stdout: ** DB Stats ** 2024-02-07T12:17:42.678 INFO:journalctl@ceph.mon.a.smithi019.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-02-07T12:17:42.678 INFO:journalctl@ceph.mon.a.smithi019.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-02-07T12:17:42.678 INFO:journalctl@ceph.mon.a.smithi019.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-02-07T12:17:42.678 INFO:journalctl@ceph.mon.a.smithi019.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-07T12:17:42.678 INFO:journalctl@ceph.mon.a.smithi019.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-02-07T12:17:42.678 INFO:journalctl@ceph.mon.a.smithi019.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-02-07T12:17:42.679 INFO:journalctl@ceph.mon.a.smithi019.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-07T12:17:42.679 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2024-02-07T12:17:42.679 INFO:journalctl@ceph.mon.a.smithi019.stdout: ** Compaction Stats [default] ** 2024-02-07T12:17:42.679 INFO:journalctl@ceph.mon.a.smithi019.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-07T12:17:42.679 INFO:journalctl@ceph.mon.a.smithi019.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-07T12:17:42.679 INFO:journalctl@ceph.mon.a.smithi019.stdout: L0 2/0 79.09 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 160.7 0.00 0.00 1 0.000 0 0 2024-02-07T12:17:42.679 INFO:journalctl@ceph.mon.a.smithi019.stdout: Sum 2/0 79.09 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 160.7 0.00 0.00 1 0.000 0 0 2024-02-07T12:17:42.679 INFO:journalctl@ceph.mon.a.smithi019.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 160.7 0.00 0.00 1 0.000 0 0 2024-02-07T12:17:42.679 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2024-02-07T12:17:42.679 INFO:journalctl@ceph.mon.a.smithi019.stdout: ** Compaction Stats [default] ** 2024-02-07T12:17:42.679 INFO:journalctl@ceph.mon.a.smithi019.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-07T12:17:42.679 INFO:journalctl@ceph.mon.a.smithi019.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-07T12:17:42.679 INFO:journalctl@ceph.mon.a.smithi019.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 160.7 0.00 0.00 1 0.000 0 0 2024-02-07T12:17:42.679 INFO:journalctl@ceph.mon.a.smithi019.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-02-07T12:17:42.679 INFO:journalctl@ceph.mon.a.smithi019.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-02-07T12:17:42.680 INFO:journalctl@ceph.mon.a.smithi019.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-02-07T12:17:42.680 INFO:journalctl@ceph.mon.a.smithi019.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-02-07T12:17:42.680 INFO:journalctl@ceph.mon.a.smithi019.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-02-07T12:17:42.680 INFO:journalctl@ceph.mon.a.smithi019.stdout: AddFile(Keys): cumulative 0, interval 0 2024-02-07T12:17:42.680 INFO:journalctl@ceph.mon.a.smithi019.stdout: Cumulative compaction: 0.00 GB write, 26.26 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:17:42.680 INFO:journalctl@ceph.mon.a.smithi019.stdout: Interval compaction: 0.00 GB write, 26.26 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:17:42.680 INFO:journalctl@ceph.mon.a.smithi019.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-02-07T12:17:42.680 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2024-02-07T12:17:42.680 INFO:journalctl@ceph.mon.a.smithi019.stdout: ** File Read Latency Histogram By Level [default] ** 2024-02-07T12:17:42.680 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2024-02-07T12:17:42.680 INFO:journalctl@ceph.mon.a.smithi019.stdout: ** Compaction Stats [default] ** 2024-02-07T12:17:42.680 INFO:journalctl@ceph.mon.a.smithi019.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-07T12:17:42.680 INFO:journalctl@ceph.mon.a.smithi019.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-07T12:17:42.680 INFO:journalctl@ceph.mon.a.smithi019.stdout: L0 2/0 79.09 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 160.7 0.00 0.00 1 0.000 0 0 2024-02-07T12:17:42.681 INFO:journalctl@ceph.mon.a.smithi019.stdout: Sum 2/0 79.09 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 160.7 0.00 0.00 1 0.000 0 0 2024-02-07T12:17:42.681 INFO:journalctl@ceph.mon.a.smithi019.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-02-07T12:17:42.681 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2024-02-07T12:17:42.681 INFO:journalctl@ceph.mon.a.smithi019.stdout: ** Compaction Stats [default] ** 2024-02-07T12:17:42.681 INFO:journalctl@ceph.mon.a.smithi019.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-07T12:17:42.681 INFO:journalctl@ceph.mon.a.smithi019.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-07T12:17:42.681 INFO:journalctl@ceph.mon.a.smithi019.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 160.7 0.00 0.00 1 0.000 0 0 2024-02-07T12:17:42.681 INFO:journalctl@ceph.mon.a.smithi019.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-02-07T12:17:42.681 INFO:journalctl@ceph.mon.a.smithi019.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-02-07T12:17:42.681 INFO:journalctl@ceph.mon.a.smithi019.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-02-07T12:17:42.681 INFO:journalctl@ceph.mon.a.smithi019.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-02-07T12:17:42.681 INFO:journalctl@ceph.mon.a.smithi019.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-02-07T12:17:42.681 INFO:journalctl@ceph.mon.a.smithi019.stdout: AddFile(Keys): cumulative 0, interval 0 2024-02-07T12:17:42.681 INFO:journalctl@ceph.mon.a.smithi019.stdout: Cumulative compaction: 0.00 GB write, 25.98 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:17:42.682 INFO:journalctl@ceph.mon.a.smithi019.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:17:42.682 INFO:journalctl@ceph.mon.a.smithi019.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-02-07T12:17:42.682 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2024-02-07T12:17:42.682 INFO:journalctl@ceph.mon.a.smithi019.stdout: ** File Read Latency Histogram By Level [default] ** 2024-02-07T12:17:42.682 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: starting mon.a rank 0 at public addrs [v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0] at bind addrs [v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:17:42.682 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: mon.a@-1(???) e1 preinit fsid ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:17:42.682 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: mon.a@-1(???).mds e1 new map 2024-02-07T12:17:42.682 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: mon.a@-1(???).mds e1 print_map 2024-02-07T12:17:42.682 INFO:journalctl@ceph.mon.a.smithi019.stdout: e1 2024-02-07T12:17:42.682 INFO:journalctl@ceph.mon.a.smithi019.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-02-07T12:17:42.682 INFO:journalctl@ceph.mon.a.smithi019.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-02-07T12:17:42.682 INFO:journalctl@ceph.mon.a.smithi019.stdout: legacy client fscid: -1 2024-02-07T12:17:42.682 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2024-02-07T12:17:42.682 INFO:journalctl@ceph.mon.a.smithi019.stdout: No filesystems configured 2024-02-07T12:17:42.682 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-02-07T12:17:42.683 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-02-07T12:17:42.683 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-02-07T12:17:42.683 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-02-07T12:17:42.683 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-02-07T12:17:42.683 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 bash[24555]: f3a8a050cf1d560b64ce22cb4e363436d142dcdb71fe1e93d919b7adc4d6fcd8 2024-02-07T12:17:42.683 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 systemd[1]: Started Ceph mon.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:17:42.683 INFO:teuthology.orchestra.run.smithi019.stderr:Setting mon public_network to 172.21.0.0/20 2024-02-07T12:17:42.909 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: mon.a is new leader, mons a in quorum (ranks 0) 2024-02-07T12:17:42.909 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: monmap e1: 1 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0]} 2024-02-07T12:17:42.909 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: fsmap 2024-02-07T12:17:42.909 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: osdmap e1: 0 total, 0 up, 0 in 2024-02-07T12:17:42.909 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:42 smithi019 ceph-mon[24574]: mgrmap e1: no daemons active 2024-02-07T12:17:43.729 INFO:teuthology.orchestra.run.smithi019.stderr:Wrote config to /etc/ceph/ceph.conf 2024-02-07T12:17:43.729 INFO:teuthology.orchestra.run.smithi019.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-02-07T12:17:43.729 INFO:teuthology.orchestra.run.smithi019.stderr:Creating mgr... 2024-02-07T12:17:43.730 INFO:teuthology.orchestra.run.smithi019.stderr:Verifying port 9283 ... 2024-02-07T12:17:43.899 INFO:teuthology.orchestra.run.smithi019.stderr:systemctl: Failed to reset failed state of unit ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mgr.y.service: Unit ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mgr.y.service not loaded. 2024-02-07T12:17:43.905 INFO:teuthology.orchestra.run.smithi019.stderr:systemctl: Created symlink /etc/systemd/system/ceph-ced2783c-c5b2-11ee-95b6-87774f69a715.target.wants/ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mgr.y.service → /etc/systemd/system/ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@.service. 2024-02-07T12:17:44.211 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:44 smithi019 systemd[1]: Starting Ceph mgr.y for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:17:44.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:44 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3725794916' entity='client.admin' 2024-02-07T12:17:44.481 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:44 smithi019 podman[24791]: 2024-02-07 12:17:44.17504898 +0000 UTC m=+0.018095302 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-02-07T12:17:44.481 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:44 smithi019 podman[24791]: 2024-02-07 12:17:44.293242834 +0000 UTC m=+0.136289145 container create c025f688a668eca87841aa483a02e4b9f494af3ad340e86323b6df703766a853 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y, io.openshift.expose-services=, name=centos-stream, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, GIT_CLEAN=True, version=8, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.buildah.version=1.19.8, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, distribution-scope=public, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream) 2024-02-07T12:17:44.482 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:44 smithi019 podman[24791]: 2024-02-07 12:17:44.479140596 +0000 UTC m=+0.322186913 container init c025f688a668eca87841aa483a02e4b9f494af3ad340e86323b6df703766a853 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, distribution-scope=public, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., RELEASE=HEAD, architecture=x86_64, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, vcs-type=git, version=8, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8) 2024-02-07T12:17:44.592 INFO:teuthology.orchestra.run.smithi019.stderr:systemctl: disabled 2024-02-07T12:17:44.599 INFO:teuthology.orchestra.run.smithi019.stderr:systemctl: inactive 2024-02-07T12:17:44.600 INFO:teuthology.orchestra.run.smithi019.stderr:firewalld.service is not enabled 2024-02-07T12:17:44.601 INFO:teuthology.orchestra.run.smithi019.stderr:Not possible to enable service . firewalld.service is not available 2024-02-07T12:17:44.608 INFO:teuthology.orchestra.run.smithi019.stderr:systemctl: disabled 2024-02-07T12:17:44.615 INFO:teuthology.orchestra.run.smithi019.stderr:systemctl: inactive 2024-02-07T12:17:44.616 INFO:teuthology.orchestra.run.smithi019.stderr:firewalld.service is not enabled 2024-02-07T12:17:44.616 INFO:teuthology.orchestra.run.smithi019.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-02-07T12:17:44.617 INFO:teuthology.orchestra.run.smithi019.stderr:Waiting for mgr to start... 2024-02-07T12:17:44.617 INFO:teuthology.orchestra.run.smithi019.stderr:Waiting for mgr... 2024-02-07T12:17:44.740 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:44 smithi019 podman[24791]: 2024-02-07 12:17:44.48213149 +0000 UTC m=+0.325177806 container start c025f688a668eca87841aa483a02e4b9f494af3ad340e86323b6df703766a853 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, architecture=x86_64, version=8, ceph=True) 2024-02-07T12:17:44.740 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:44 smithi019 bash[24791]: c025f688a668eca87841aa483a02e4b9f494af3ad340e86323b6df703766a853 2024-02-07T12:17:44.740 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:44 smithi019 systemd[1]: Started Ceph mgr.y for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:17:45.078 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:44 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:17:44.981+0000 7f584bab6000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-02-07T12:17:45.078 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:45 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:17:45.075+0000 7f584bab6000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-02-07T12:17:45.169 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: 2024-02-07T12:17:45.169 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: { 2024-02-07T12:17:45.169 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "fsid": "ced2783c-c5b2-11ee-95b6-87774f69a715", 2024-02-07T12:17:45.170 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "health": { 2024-02-07T12:17:45.170 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-02-07T12:17:45.170 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "checks": {}, 2024-02-07T12:17:45.170 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "mutes": [] 2024-02-07T12:17:45.170 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: }, 2024-02-07T12:17:45.170 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-02-07T12:17:45.170 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "quorum": [ 2024-02-07T12:17:45.170 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: 0 2024-02-07T12:17:45.170 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: ], 2024-02-07T12:17:45.170 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "quorum_names": [ 2024-02-07T12:17:45.170 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "a" 2024-02-07T12:17:45.170 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: ], 2024-02-07T12:17:45.170 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-02-07T12:17:45.171 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "monmap": { 2024-02-07T12:17:45.171 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-07T12:17:45.171 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-02-07T12:17:45.171 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_mons": 1 2024-02-07T12:17:45.171 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: }, 2024-02-07T12:17:45.171 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "osdmap": { 2024-02-07T12:17:45.171 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-07T12:17:45.171 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_osds": 0, 2024-02-07T12:17:45.171 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-02-07T12:17:45.171 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-02-07T12:17:45.171 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-02-07T12:17:45.171 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-02-07T12:17:45.171 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-02-07T12:17:45.171 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: }, 2024-02-07T12:17:45.171 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "pgmap": { 2024-02-07T12:17:45.171 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-02-07T12:17:45.172 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-02-07T12:17:45.172 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_pools": 0, 2024-02-07T12:17:45.172 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_objects": 0, 2024-02-07T12:17:45.172 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-02-07T12:17:45.172 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-02-07T12:17:45.172 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-02-07T12:17:45.172 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "bytes_total": 0 2024-02-07T12:17:45.172 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: }, 2024-02-07T12:17:45.172 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "fsmap": { 2024-02-07T12:17:45.172 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-07T12:17:45.172 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "by_rank": [], 2024-02-07T12:17:45.172 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "up:standby": 0 2024-02-07T12:17:45.172 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: }, 2024-02-07T12:17:45.172 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "mgrmap": { 2024-02-07T12:17:45.173 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "available": false, 2024-02-07T12:17:45.173 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-02-07T12:17:45.173 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "modules": [ 2024-02-07T12:17:45.173 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "iostat", 2024-02-07T12:17:45.173 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "nfs", 2024-02-07T12:17:45.173 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "restful" 2024-02-07T12:17:45.173 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: ], 2024-02-07T12:17:45.173 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "services": {} 2024-02-07T12:17:45.173 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: }, 2024-02-07T12:17:45.174 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "servicemap": { 2024-02-07T12:17:45.174 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-07T12:17:45.174 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "modified": "2024-02-07T12:17:38.251490+0000", 2024-02-07T12:17:45.174 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "services": {} 2024-02-07T12:17:45.174 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: }, 2024-02-07T12:17:45.174 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "progress_events": {} 2024-02-07T12:17:45.174 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: } 2024-02-07T12:17:45.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:45 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2970286940' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-02-07T12:17:45.630 INFO:teuthology.orchestra.run.smithi019.stderr:mgr not available, waiting (1/15)... 2024-02-07T12:17:46.036 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:45 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:17:45.706+0000 7f584bab6000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-02-07T12:17:46.036 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:45 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:17:45.797+0000 7f584bab6000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-02-07T12:17:46.036 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:45 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:17:45.882+0000 7f584bab6000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-02-07T12:17:46.314 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:46 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:17:46.033+0000 7f584bab6000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-02-07T12:17:46.314 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:46 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:17:46.115+0000 7f584bab6000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-02-07T12:17:46.314 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:46 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:17:46.226+0000 7f584bab6000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-02-07T12:17:46.314 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:46 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:17:46.311+0000 7f584bab6000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-02-07T12:17:46.565 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:46 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:17:46.475+0000 7f584bab6000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-02-07T12:17:46.565 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:46 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:17:46.562+0000 7f584bab6000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-02-07T12:17:47.890 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:17:47.616+0000 7f584bab6000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-02-07T12:17:47.890 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:17:47.757+0000 7f584bab6000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-02-07T12:17:48.146 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:17:47.954+0000 7f584bab6000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-02-07T12:17:48.146 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:48 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:17:48.045+0000 7f584bab6000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-02-07T12:17:48.149 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: 2024-02-07T12:17:48.149 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: { 2024-02-07T12:17:48.150 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "fsid": "ced2783c-c5b2-11ee-95b6-87774f69a715", 2024-02-07T12:17:48.150 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "health": { 2024-02-07T12:17:48.150 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-02-07T12:17:48.150 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "checks": {}, 2024-02-07T12:17:48.150 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "mutes": [] 2024-02-07T12:17:48.150 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: }, 2024-02-07T12:17:48.150 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-02-07T12:17:48.150 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "quorum": [ 2024-02-07T12:17:48.150 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: 0 2024-02-07T12:17:48.150 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: ], 2024-02-07T12:17:48.150 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "quorum_names": [ 2024-02-07T12:17:48.150 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "a" 2024-02-07T12:17:48.150 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: ], 2024-02-07T12:17:48.150 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-02-07T12:17:48.150 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "monmap": { 2024-02-07T12:17:48.151 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-07T12:17:48.151 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-02-07T12:17:48.151 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_mons": 1 2024-02-07T12:17:48.151 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: }, 2024-02-07T12:17:48.151 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "osdmap": { 2024-02-07T12:17:48.151 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-07T12:17:48.151 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_osds": 0, 2024-02-07T12:17:48.151 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-02-07T12:17:48.151 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-02-07T12:17:48.151 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-02-07T12:17:48.154 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-02-07T12:17:48.154 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-02-07T12:17:48.154 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: }, 2024-02-07T12:17:48.154 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "pgmap": { 2024-02-07T12:17:48.155 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-02-07T12:17:48.155 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-02-07T12:17:48.155 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_pools": 0, 2024-02-07T12:17:48.155 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_objects": 0, 2024-02-07T12:17:48.155 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-02-07T12:17:48.155 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-02-07T12:17:48.155 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-02-07T12:17:48.155 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "bytes_total": 0 2024-02-07T12:17:48.155 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: }, 2024-02-07T12:17:48.155 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "fsmap": { 2024-02-07T12:17:48.155 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-07T12:17:48.156 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "by_rank": [], 2024-02-07T12:17:48.156 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "up:standby": 0 2024-02-07T12:17:48.156 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: }, 2024-02-07T12:17:48.156 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "mgrmap": { 2024-02-07T12:17:48.156 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "available": false, 2024-02-07T12:17:48.156 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-02-07T12:17:48.156 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "modules": [ 2024-02-07T12:17:48.156 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "iostat", 2024-02-07T12:17:48.156 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "nfs", 2024-02-07T12:17:48.156 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "restful" 2024-02-07T12:17:48.156 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: ], 2024-02-07T12:17:48.156 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "services": {} 2024-02-07T12:17:48.156 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: }, 2024-02-07T12:17:48.156 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "servicemap": { 2024-02-07T12:17:48.156 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-07T12:17:48.157 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "modified": "2024-02-07T12:17:38.251490+0000", 2024-02-07T12:17:48.157 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "services": {} 2024-02-07T12:17:48.157 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: }, 2024-02-07T12:17:48.157 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "progress_events": {} 2024-02-07T12:17:48.157 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: } 2024-02-07T12:17:48.521 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:48 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:17:48.196+0000 7f584bab6000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:17:48.521 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:48 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:17:48.518+0000 7f584bab6000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-02-07T12:17:48.834 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:48 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/575747305' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-02-07T12:17:48.834 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:48 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:17:48.744+0000 7f584bab6000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:17:48.834 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:48 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:17:48.831+0000 7f584bab6000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-02-07T12:17:49.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:49 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:17:49.006+0000 7f584bab6000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-02-07T12:17:50.256 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:49 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:17:49.949+0000 7f584bab6000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-02-07T12:17:50.570 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:17:50.254+0000 7f584bab6000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-02-07T12:17:50.570 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:17:50.350+0000 7f584bab6000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-02-07T12:17:50.570 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:17:50.504+0000 7f584bab6000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-02-07T12:17:50.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:50 smithi019 ceph-mon[24574]: Activating manager daemon y 2024-02-07T12:17:50.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:50 smithi019 ceph-mon[24574]: mgrmap e2: y(active, starting, since 0.00279267s) 2024-02-07T12:17:50.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:50 smithi019 ceph-mon[24574]: from='mgr.14100 172.21.15.19:0/2085214275' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:17:50.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:50 smithi019 ceph-mon[24574]: from='mgr.14100 172.21.15.19:0/2085214275' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:17:50.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:50 smithi019 ceph-mon[24574]: from='mgr.14100 172.21.15.19:0/2085214275' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:17:50.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:50 smithi019 ceph-mon[24574]: from='mgr.14100 172.21.15.19:0/2085214275' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:17:50.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:50 smithi019 ceph-mon[24574]: from='mgr.14100 172.21.15.19:0/2085214275' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-02-07T12:17:50.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:50 smithi019 ceph-mon[24574]: Manager daemon y is now available 2024-02-07T12:17:50.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:50 smithi019 ceph-mon[24574]: from='mgr.14100 172.21.15.19:0/2085214275' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:17:50.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:50 smithi019 ceph-mon[24574]: from='mgr.14100 172.21.15.19:0/2085214275' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:17:50.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:50 smithi019 ceph-mon[24574]: from='mgr.14100 172.21.15.19:0/2085214275' entity='mgr.y' 2024-02-07T12:17:50.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:50 smithi019 ceph-mon[24574]: from='mgr.14100 172.21.15.19:0/2085214275' entity='mgr.y' 2024-02-07T12:17:50.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:50 smithi019 ceph-mon[24574]: from='mgr.14100 172.21.15.19:0/2085214275' entity='mgr.y' 2024-02-07T12:17:51.302 INFO:teuthology.orchestra.run.smithi019.stderr:mgr not available, waiting (2/15)... 2024-02-07T12:17:52.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:52 smithi019 ceph-mon[24574]: mgrmap e3: y(active, since 1.00624s) 2024-02-07T12:17:54.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:54 smithi019 ceph-mon[24574]: mgrmap e4: y(active, since 3s) 2024-02-07T12:17:57.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:56 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/4091380382' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-02-07T12:17:57.187 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: 2024-02-07T12:17:57.187 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: { 2024-02-07T12:17:57.187 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "fsid": "ced2783c-c5b2-11ee-95b6-87774f69a715", 2024-02-07T12:17:57.187 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "health": { 2024-02-07T12:17:57.187 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-02-07T12:17:57.188 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "checks": {}, 2024-02-07T12:17:57.188 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "mutes": [] 2024-02-07T12:17:57.188 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: }, 2024-02-07T12:17:57.188 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-02-07T12:17:57.188 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "quorum": [ 2024-02-07T12:17:57.188 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: 0 2024-02-07T12:17:57.188 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: ], 2024-02-07T12:17:57.188 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "quorum_names": [ 2024-02-07T12:17:57.188 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "a" 2024-02-07T12:17:57.188 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: ], 2024-02-07T12:17:57.188 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "quorum_age": 14, 2024-02-07T12:17:57.189 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "monmap": { 2024-02-07T12:17:57.189 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-07T12:17:57.189 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-02-07T12:17:57.189 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_mons": 1 2024-02-07T12:17:57.189 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: }, 2024-02-07T12:17:57.189 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "osdmap": { 2024-02-07T12:17:57.189 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-07T12:17:57.189 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_osds": 0, 2024-02-07T12:17:57.189 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-02-07T12:17:57.189 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-02-07T12:17:57.189 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-02-07T12:17:57.193 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-02-07T12:17:57.193 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-02-07T12:17:57.193 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: }, 2024-02-07T12:17:57.193 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "pgmap": { 2024-02-07T12:17:57.193 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-02-07T12:17:57.194 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-02-07T12:17:57.194 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_pools": 0, 2024-02-07T12:17:57.194 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_objects": 0, 2024-02-07T12:17:57.194 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-02-07T12:17:57.194 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-02-07T12:17:57.194 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-02-07T12:17:57.194 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "bytes_total": 0 2024-02-07T12:17:57.194 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: }, 2024-02-07T12:17:57.194 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "fsmap": { 2024-02-07T12:17:57.194 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-07T12:17:57.194 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "by_rank": [], 2024-02-07T12:17:57.194 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "up:standby": 0 2024-02-07T12:17:57.194 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: }, 2024-02-07T12:17:57.195 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "mgrmap": { 2024-02-07T12:17:57.195 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "available": true, 2024-02-07T12:17:57.195 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-02-07T12:17:57.195 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "modules": [ 2024-02-07T12:17:57.195 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "iostat", 2024-02-07T12:17:57.195 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "nfs", 2024-02-07T12:17:57.195 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "restful" 2024-02-07T12:17:57.195 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: ], 2024-02-07T12:17:57.195 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "services": {} 2024-02-07T12:17:57.195 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: }, 2024-02-07T12:17:57.196 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "servicemap": { 2024-02-07T12:17:57.196 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-07T12:17:57.196 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "modified": "2024-02-07T12:17:38.251490+0000", 2024-02-07T12:17:57.196 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "services": {} 2024-02-07T12:17:57.197 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: }, 2024-02-07T12:17:57.197 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "progress_events": {} 2024-02-07T12:17:57.197 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: } 2024-02-07T12:17:58.058 INFO:teuthology.orchestra.run.smithi019.stderr:mgr is available 2024-02-07T12:17:58.699 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: 2024-02-07T12:17:58.700 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: [global] 2024-02-07T12:17:58.701 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: fsid = ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:17:58.701 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-02-07T12:17:58.701 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-02-07T12:17:58.701 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-02-07T12:17:58.701 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-02-07T12:17:58.701 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: 2024-02-07T12:17:58.701 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: [mgr] 2024-02-07T12:17:58.701 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-02-07T12:17:58.701 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: 2024-02-07T12:17:58.701 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: [osd] 2024-02-07T12:17:58.701 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-02-07T12:17:58.701 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-02-07T12:17:59.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:58 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3503193606' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-02-07T12:17:59.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:58 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3503193606' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-02-07T12:17:59.190 INFO:teuthology.orchestra.run.smithi019.stderr:Enabling cephadm module... 2024-02-07T12:18:00.176 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:17:59 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3994683952' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-02-07T12:18:00.177 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:59 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ignoring --setuser ceph since I am not root 2024-02-07T12:18:00.177 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:17:59 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ignoring --setgroup ceph since I am not root 2024-02-07T12:18:00.493 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:00.395+0000 7fc3e1afa000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-02-07T12:18:00.493 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:00.490+0000 7fc3e1afa000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-02-07T12:18:01.063 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: { 2024-02-07T12:18:01.063 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "epoch": 5, 2024-02-07T12:18:01.063 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "available": true, 2024-02-07T12:18:01.063 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "active_name": "y", 2024-02-07T12:18:01.063 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_standby": 0 2024-02-07T12:18:01.063 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: } 2024-02-07T12:18:01.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:00 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3994683952' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-02-07T12:18:01.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:00 smithi019 ceph-mon[24574]: mgrmap e5: y(active, since 9s) 2024-02-07T12:18:01.416 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:01 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:01.110+0000 7fc3e1afa000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-02-07T12:18:01.416 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:01 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:01.201+0000 7fc3e1afa000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-02-07T12:18:01.416 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:01 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:01.286+0000 7fc3e1afa000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-02-07T12:18:01.721 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:01 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:01.439+0000 7fc3e1afa000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-02-07T12:18:01.721 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:01 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:01.521+0000 7fc3e1afa000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-02-07T12:18:01.721 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:01 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:01.633+0000 7fc3e1afa000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-02-07T12:18:01.721 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:01 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:01.718+0000 7fc3e1afa000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-02-07T12:18:01.973 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:01 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3201875467' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-02-07T12:18:01.973 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:01 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:01.883+0000 7fc3e1afa000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-02-07T12:18:01.973 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:01 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:01.971+0000 7fc3e1afa000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-02-07T12:18:02.227 INFO:teuthology.orchestra.run.smithi019.stderr:Waiting for the mgr to restart... 2024-02-07T12:18:02.228 INFO:teuthology.orchestra.run.smithi019.stderr:Waiting for mgr epoch 5... 2024-02-07T12:18:03.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:03.031+0000 7fc3e1afa000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-02-07T12:18:03.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:03.167+0000 7fc3e1afa000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-02-07T12:18:03.844 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:03.362+0000 7fc3e1afa000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-02-07T12:18:03.844 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:03.452+0000 7fc3e1afa000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-02-07T12:18:03.844 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:03.591+0000 7fc3e1afa000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:18:04.225 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:03.912+0000 7fc3e1afa000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-02-07T12:18:04.225 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:04 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:04.136+0000 7fc3e1afa000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:18:04.225 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:04 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:04.222+0000 7fc3e1afa000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-02-07T12:18:04.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:04 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:04.396+0000 7fc3e1afa000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-02-07T12:18:05.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:05 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:05.336+0000 7fc3e1afa000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-02-07T12:18:05.894 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:05 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:05.638+0000 7fc3e1afa000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-02-07T12:18:05.894 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:05 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:05.736+0000 7fc3e1afa000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-02-07T12:18:05.894 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:05 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:05.892+0000 7fc3e1afa000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-02-07T12:18:06.973 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: { 2024-02-07T12:18:06.974 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-02-07T12:18:06.974 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "initialized": true 2024-02-07T12:18:06.974 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: } 2024-02-07T12:18:07.272 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:06 smithi019 ceph-mon[24574]: Active manager daemon y restarted 2024-02-07T12:18:07.273 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:06 smithi019 ceph-mon[24574]: Activating manager daemon y 2024-02-07T12:18:07.273 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:06 smithi019 ceph-mon[24574]: osdmap e2: 0 total, 0 up, 0 in 2024-02-07T12:18:07.273 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:06 smithi019 ceph-mon[24574]: mgrmap e6: y(active, starting, since 0.0525618s) 2024-02-07T12:18:07.273 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:06 smithi019 ceph-mon[24574]: from='mgr.14118 172.21.15.19:0/3651063590' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:18:07.273 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:06 smithi019 ceph-mon[24574]: from='mgr.14118 172.21.15.19:0/3651063590' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-02-07T12:18:07.273 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:06 smithi019 ceph-mon[24574]: from='mgr.14118 172.21.15.19:0/3651063590' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:18:07.273 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:06 smithi019 ceph-mon[24574]: from='mgr.14118 172.21.15.19:0/3651063590' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:18:07.273 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:06 smithi019 ceph-mon[24574]: from='mgr.14118 172.21.15.19:0/3651063590' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:18:07.273 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:06 smithi019 ceph-mon[24574]: Manager daemon y is now available 2024-02-07T12:18:07.273 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:06 smithi019 ceph-mon[24574]: from='mgr.14118 172.21.15.19:0/3651063590' entity='mgr.y' 2024-02-07T12:18:07.273 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:06 smithi019 ceph-mon[24574]: from='mgr.14118 172.21.15.19:0/3651063590' entity='mgr.y' 2024-02-07T12:18:07.273 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:06 smithi019 ceph-mon[24574]: from='mgr.14118 172.21.15.19:0/3651063590' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:18:07.274 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:06 smithi019 ceph-mon[24574]: from='mgr.14118 172.21.15.19:0/3651063590' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:18:07.274 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:06 smithi019 ceph-mon[24574]: from='mgr.14118 172.21.15.19:0/3651063590' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:18:07.274 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:06 smithi019 ceph-mon[24574]: from='mgr.14118 172.21.15.19:0/3651063590' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:18:07.274 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:06 smithi019 ceph-mon[24574]: from='mgr.14118 172.21.15.19:0/3651063590' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:18:07.537 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: [07/Feb/2024:12:18:07] ENGINE Bus STARTING 2024-02-07T12:18:07.539 INFO:teuthology.orchestra.run.smithi019.stderr:mgr epoch 5 is available 2024-02-07T12:18:07.540 INFO:teuthology.orchestra.run.smithi019.stderr:Setting orchestrator backend to cephadm... 2024-02-07T12:18:07.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: [07/Feb/2024:12:18:07] ENGINE Serving on https://172.21.15.19:7150 2024-02-07T12:18:07.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: [07/Feb/2024:12:18:07] ENGINE Bus STARTED 2024-02-07T12:18:08.163 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:07 smithi019 ceph-mon[24574]: mgrmap e7: y(active, since 1.05503s) 2024-02-07T12:18:08.163 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:07 smithi019 ceph-mon[24574]: [07/Feb/2024:12:18:07] ENGINE Bus STARTING 2024-02-07T12:18:08.163 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:07 smithi019 ceph-mon[24574]: [07/Feb/2024:12:18:07] ENGINE Serving on https://172.21.15.19:7150 2024-02-07T12:18:08.163 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:07 smithi019 ceph-mon[24574]: [07/Feb/2024:12:18:07] ENGINE Bus STARTED 2024-02-07T12:18:08.163 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:07 smithi019 ceph-mon[24574]: from='mgr.14118 172.21.15.19:0/3651063590' entity='mgr.y' 2024-02-07T12:18:08.163 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:07 smithi019 ceph-mon[24574]: from='mgr.14118 172.21.15.19:0/3651063590' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:18:09.327 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:09 smithi019 ceph-mon[24574]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:18:09.327 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:09 smithi019 ceph-mon[24574]: from='mgr.14118 172.21.15.19:0/3651063590' entity='mgr.y' 2024-02-07T12:18:09.327 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:09 smithi019 ceph-mon[24574]: from='mgr.14118 172.21.15.19:0/3651063590' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:18:09.327 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:09 smithi019 ceph-mon[24574]: mgrmap e8: y(active, since 2s) 2024-02-07T12:18:09.328 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: value unchanged 2024-02-07T12:18:09.852 INFO:teuthology.orchestra.run.smithi019.stderr:Generating ssh key... 2024-02-07T12:18:10.210 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:10 smithi019 ceph-mon[24574]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:18:10.972 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: Generating public/private rsa key pair. 2024-02-07T12:18:10.972 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: Your identification has been saved in /tmp/tmpjr1xe394/key. 2024-02-07T12:18:10.972 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: Your public key has been saved in /tmp/tmpjr1xe394/key.pub. 2024-02-07T12:18:10.972 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: The key fingerprint is: 2024-02-07T12:18:10.972 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: SHA256:YZM0W1NrrV5n1CCTmTXZCVsoPG5dTDMZ+wlNnL+HtcI ceph-ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:18:10.972 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: The key's randomart image is: 2024-02-07T12:18:10.972 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: +---[RSA 3072]----+ 2024-02-07T12:18:10.972 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: | o +.+*XX=| 2024-02-07T12:18:10.972 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: | . = =+**X*| 2024-02-07T12:18:10.973 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: | * . *ooo+| 2024-02-07T12:18:10.973 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: | . o + o..=| 2024-02-07T12:18:10.973 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: | S . o .+*| 2024-02-07T12:18:10.973 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: | . E.=.| 2024-02-07T12:18:10.973 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: | . . .| 2024-02-07T12:18:10.973 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: | | 2024-02-07T12:18:10.973 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: | | 2024-02-07T12:18:10.973 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: +----[SHA256]-----+ 2024-02-07T12:18:11.925 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:11 smithi019 ceph-mon[24574]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:18:11.925 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:11 smithi019 ceph-mon[24574]: Generating ssh key... 2024-02-07T12:18:11.925 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:11 smithi019 ceph-mon[24574]: from='mgr.14118 172.21.15.19:0/3651063590' entity='mgr.y' 2024-02-07T12:18:11.925 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:11 smithi019 ceph-mon[24574]: from='mgr.14118 172.21.15.19:0/3651063590' entity='mgr.y' 2024-02-07T12:18:12.132 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC87Juc203zMsHCynIvpzKfCBp8d2JApSBh9ab8ICI3mPQ+QRE0xvluP14MQEKrrJA/pDRCMkPjCgrAacyjvR937OyBh+/QMP47lOhF+NJYSfFjQ1ssmq47CA/e5GmfR6SAxyaYWPMrlahb/RnMk6i8xT9hAEJmN/GOF0/KWwpC/yD+0ezo4p5NBGJhPDHN0DO2gqAbMt3G70Vcuhivimdb//e8GyKMru+nAA6SLwVO12BleGMDUcPl4c6oNK37V70LmMrKkwDdHxGgaGbT+YA0pB9R6aTTY/X7XA1Lxrg9sZlf6+ADJ4GtaKHiUpujLs0iykJGmUD9nzMJzz66BNRd5NV9zLRJQG9Q52Kcw01bAqN1odUBAM/BdhAKeoAO69kAlQfoR5KkjDObyZLWS8ybCzfZXWQkEMJkp98ZOY93CA3B1oDRK97udH6PLFwpSfHGTjY1yZWahNFBo/eRDUpmblHbfQnTMsNWsUaCqv8DaRZPaSXRXPX4srPzNGGU9yc= ceph-ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:18:12.690 INFO:teuthology.orchestra.run.smithi019.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-02-07T12:18:12.690 INFO:teuthology.orchestra.run.smithi019.stderr:Adding key to root@localhost authorized_keys... 2024-02-07T12:18:12.691 INFO:teuthology.orchestra.run.smithi019.stderr:Adding host smithi019... 2024-02-07T12:18:14.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:13 smithi019 ceph-mon[24574]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:18:14.354 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: Added host 'smithi019' with addr '172.21.15.19' 2024-02-07T12:18:14.953 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:14 smithi019 ceph-mon[24574]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi019", "addr": "172.21.15.19", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:18:14.953 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:14 smithi019 ceph-mon[24574]: Deploying cephadm binary to smithi019 2024-02-07T12:18:14.953 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:14 smithi019 ceph-mon[24574]: from='mgr.14118 172.21.15.19:0/3651063590' entity='mgr.y' 2024-02-07T12:18:14.953 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:14 smithi019 ceph-mon[24574]: from='mgr.14118 172.21.15.19:0/3651063590' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:18:14.955 INFO:teuthology.orchestra.run.smithi019.stderr:Deploying unmanaged mon service... 2024-02-07T12:18:15.639 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: Scheduled mon update... 2024-02-07T12:18:15.896 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:15 smithi019 ceph-mon[24574]: Added host smithi019 2024-02-07T12:18:16.508 INFO:teuthology.orchestra.run.smithi019.stderr:Deploying unmanaged mgr service... 2024-02-07T12:18:16.811 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:16 smithi019 ceph-mon[24574]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:18:16.811 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:16 smithi019 ceph-mon[24574]: Saving service mon spec with placement count:5 2024-02-07T12:18:16.811 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:16 smithi019 ceph-mon[24574]: from='mgr.14118 172.21.15.19:0/3651063590' entity='mgr.y' 2024-02-07T12:18:17.318 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-02-07T12:18:18.437 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:18 smithi019 ceph-mon[24574]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:18:18.437 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:18 smithi019 ceph-mon[24574]: Saving service mgr spec with placement count:2 2024-02-07T12:18:18.437 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:18 smithi019 ceph-mon[24574]: from='mgr.14118 172.21.15.19:0/3651063590' entity='mgr.y' 2024-02-07T12:18:18.437 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:18 smithi019 ceph-mon[24574]: from='mgr.14118 172.21.15.19:0/3651063590' entity='mgr.y' 2024-02-07T12:18:18.437 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:18 smithi019 ceph-mon[24574]: from='mgr.14118 172.21.15.19:0/3651063590' entity='mgr.y' 2024-02-07T12:18:20.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:19 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3215981994' entity='client.admin' 2024-02-07T12:18:20.895 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:20 smithi019 ceph-mon[24574]: from='mgr.14118 172.21.15.19:0/3651063590' entity='mgr.y' 2024-02-07T12:18:20.895 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:20 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2223887045' entity='client.admin' 2024-02-07T12:18:20.895 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:20 smithi019 ceph-mon[24574]: from='mgr.14118 172.21.15.19:0/3651063590' entity='mgr.y' 2024-02-07T12:18:20.951 INFO:teuthology.orchestra.run.smithi019.stderr:Enabling the dashboard module... 2024-02-07T12:18:22.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:21 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3624756722' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-02-07T12:18:22.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:21 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ignoring --setuser ceph since I am not root 2024-02-07T12:18:22.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:21 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ignoring --setgroup ceph since I am not root 2024-02-07T12:18:22.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:22 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:22.336+0000 7f2cd4d7d000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-02-07T12:18:22.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:22 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:22.427+0000 7f2cd4d7d000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-02-07T12:18:22.937 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:22 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3624756722' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-02-07T12:18:22.937 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:22 smithi019 ceph-mon[24574]: mgrmap e9: y(active, since 15s) 2024-02-07T12:18:23.225 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:23.043+0000 7f2cd4d7d000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-02-07T12:18:23.225 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:23.136+0000 7f2cd4d7d000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-02-07T12:18:23.225 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:23.223+0000 7f2cd4d7d000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-02-07T12:18:23.373 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: { 2024-02-07T12:18:23.373 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "epoch": 9, 2024-02-07T12:18:23.373 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "available": true, 2024-02-07T12:18:23.373 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "active_name": "y", 2024-02-07T12:18:23.373 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "num_standby": 0 2024-02-07T12:18:23.373 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: } 2024-02-07T12:18:23.497 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:23.376+0000 7f2cd4d7d000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-02-07T12:18:23.497 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:23.462+0000 7f2cd4d7d000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-02-07T12:18:23.828 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:23.575+0000 7f2cd4d7d000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-02-07T12:18:23.828 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:23.660+0000 7f2cd4d7d000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-02-07T12:18:23.828 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:23.826+0000 7f2cd4d7d000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-02-07T12:18:24.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:23 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3745321853' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-02-07T12:18:24.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:23.914+0000 7f2cd4d7d000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-02-07T12:18:24.121 INFO:teuthology.orchestra.run.smithi019.stderr:Waiting for the mgr to restart... 2024-02-07T12:18:24.121 INFO:teuthology.orchestra.run.smithi019.stderr:Waiting for mgr epoch 9... 2024-02-07T12:18:25.298 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:24 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:24.970+0000 7f2cd4d7d000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-02-07T12:18:25.298 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:25 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:25.102+0000 7f2cd4d7d000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-02-07T12:18:25.298 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:25 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:25.295+0000 7f2cd4d7d000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-02-07T12:18:25.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:25 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:25.384+0000 7f2cd4d7d000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-02-07T12:18:25.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:25 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:25.522+0000 7f2cd4d7d000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:18:26.157 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:25 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:25.843+0000 7f2cd4d7d000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-02-07T12:18:26.157 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:26 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:26.069+0000 7f2cd4d7d000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:18:26.158 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:26 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:26.155+0000 7f2cd4d7d000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-02-07T12:18:26.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:26 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:26.329+0000 7f2cd4d7d000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-02-07T12:18:27.570 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:27 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:27.263+0000 7f2cd4d7d000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-02-07T12:18:27.570 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:27 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:27.568+0000 7f2cd4d7d000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-02-07T12:18:27.822 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:27 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:27.664+0000 7f2cd4d7d000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-02-07T12:18:28.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:27 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:18:27.820+0000 7f2cd4d7d000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-02-07T12:18:28.995 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:28 smithi019 ceph-mon[24574]: Active manager daemon y restarted 2024-02-07T12:18:28.995 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:28 smithi019 ceph-mon[24574]: Activating manager daemon y 2024-02-07T12:18:28.995 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:28 smithi019 ceph-mon[24574]: osdmap e3: 0 total, 0 up, 0 in 2024-02-07T12:18:28.995 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:28 smithi019 ceph-mon[24574]: mgrmap e10: y(active, starting, since 0.78864s) 2024-02-07T12:18:28.995 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:28 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:18:28.996 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:28 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-02-07T12:18:28.996 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:28 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:18:28.996 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:28 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:18:28.996 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:28 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:18:28.996 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:28 smithi019 ceph-mon[24574]: Manager daemon y is now available 2024-02-07T12:18:28.996 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:28 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:18:28.996 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:28 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:18:28.996 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:28 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:18:28.996 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:28 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:18:29.500 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:29 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: [07/Feb/2024:12:18:29] ENGINE Bus STARTING 2024-02-07T12:18:29.500 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:29 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: [07/Feb/2024:12:18:29] ENGINE Serving on https://172.21.15.19:7150 2024-02-07T12:18:29.500 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:18:29 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: [07/Feb/2024:12:18:29] ENGINE Bus STARTED 2024-02-07T12:18:29.635 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: { 2024-02-07T12:18:29.635 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-02-07T12:18:29.635 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: "initialized": true 2024-02-07T12:18:29.636 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: } 2024-02-07T12:18:30.356 INFO:teuthology.orchestra.run.smithi019.stderr:mgr epoch 9 is available 2024-02-07T12:18:30.356 INFO:teuthology.orchestra.run.smithi019.stderr:Generating a dashboard self-signed certificate... 2024-02-07T12:18:30.432 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:30 smithi019 ceph-mon[24574]: [07/Feb/2024:12:18:29] ENGINE Bus STARTING 2024-02-07T12:18:30.432 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:30 smithi019 ceph-mon[24574]: [07/Feb/2024:12:18:29] ENGINE Serving on https://172.21.15.19:7150 2024-02-07T12:18:30.432 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:30 smithi019 ceph-mon[24574]: [07/Feb/2024:12:18:29] ENGINE Bus STARTED 2024-02-07T12:18:30.432 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:30 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:18:30.432 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:30 smithi019 ceph-mon[24574]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-02-07T12:18:30.432 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:30 smithi019 ceph-mon[24574]: mgrmap e11: y(active, since 1.79342s) 2024-02-07T12:18:30.432 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:30 smithi019 ceph-mon[24574]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-02-07T12:18:31.121 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: Self-signed certificate created 2024-02-07T12:18:31.735 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:31 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:18:31.735 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:31 smithi019 ceph-mon[24574]: mgrmap e12: y(active, since 2s) 2024-02-07T12:18:31.735 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:31 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:18:31.736 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:31 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:18:31.736 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:31 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:18:31.737 INFO:teuthology.orchestra.run.smithi019.stderr:Creating initial admin user... 2024-02-07T12:18:32.583 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:32 smithi019 ceph-mon[24574]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:18:32.667 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$lg20ZlLoir.FpGoNkCCxTujM9GqusEPuQeEk.9QruQZWYqjcdmKTq", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1707308312, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-02-07T12:18:33.499 INFO:teuthology.orchestra.run.smithi019.stderr:Fetching dashboard port number... 2024-02-07T12:18:33.736 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:33 smithi019 ceph-mon[24574]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:18:33.736 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:33 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:18:34.146 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: 8443 2024-02-07T12:18:34.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:34 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3665063554' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-02-07T12:18:35.019 INFO:teuthology.orchestra.run.smithi019.stderr:systemctl: disabled 2024-02-07T12:18:35.026 INFO:teuthology.orchestra.run.smithi019.stderr:systemctl: inactive 2024-02-07T12:18:35.026 INFO:teuthology.orchestra.run.smithi019.stderr:firewalld.service is not enabled 2024-02-07T12:18:35.026 INFO:teuthology.orchestra.run.smithi019.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-02-07T12:18:35.029 INFO:teuthology.orchestra.run.smithi019.stderr:Ceph Dashboard is now available at: 2024-02-07T12:18:35.029 INFO:teuthology.orchestra.run.smithi019.stderr: 2024-02-07T12:18:35.029 INFO:teuthology.orchestra.run.smithi019.stderr: URL: https://smithi019.front.sepia.ceph.com:8443/ 2024-02-07T12:18:35.029 INFO:teuthology.orchestra.run.smithi019.stderr: User: admin 2024-02-07T12:18:35.029 INFO:teuthology.orchestra.run.smithi019.stderr: Password: k3p40dab74 2024-02-07T12:18:35.029 INFO:teuthology.orchestra.run.smithi019.stderr: 2024-02-07T12:18:35.031 INFO:teuthology.orchestra.run.smithi019.stderr:Enabling autotune for osd_memory_target 2024-02-07T12:18:36.054 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:35 smithi019 ceph-mon[24574]: mgrmap e13: y(active, since 6s) 2024-02-07T12:18:37.131 INFO:teuthology.orchestra.run.smithi019.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-02-07T12:18:37.815 INFO:teuthology.orchestra.run.smithi019.stderr:You can access the Ceph CLI with: 2024-02-07T12:18:37.815 INFO:teuthology.orchestra.run.smithi019.stderr: 2024-02-07T12:18:37.816 INFO:teuthology.orchestra.run.smithi019.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-02-07T12:18:37.816 INFO:teuthology.orchestra.run.smithi019.stderr: 2024-02-07T12:18:37.816 INFO:teuthology.orchestra.run.smithi019.stderr:Please consider enabling telemetry to help improve Ceph: 2024-02-07T12:18:37.816 INFO:teuthology.orchestra.run.smithi019.stderr: 2024-02-07T12:18:37.816 INFO:teuthology.orchestra.run.smithi019.stderr: ceph telemetry on 2024-02-07T12:18:37.816 INFO:teuthology.orchestra.run.smithi019.stderr: 2024-02-07T12:18:37.816 INFO:teuthology.orchestra.run.smithi019.stderr:For more information see: 2024-02-07T12:18:37.816 INFO:teuthology.orchestra.run.smithi019.stderr: 2024-02-07T12:18:37.816 INFO:teuthology.orchestra.run.smithi019.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-02-07T12:18:37.816 INFO:teuthology.orchestra.run.smithi019.stderr: 2024-02-07T12:18:37.816 INFO:teuthology.orchestra.run.smithi019.stderr:Bootstrap complete. 2024-02-07T12:18:37.863 INFO:tasks.cephadm:Fetching config... 2024-02-07T12:18:37.864 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2024-02-07T12:18:37.864 DEBUG:teuthology.orchestra.run.smithi019:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-02-07T12:18:37.881 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-02-07T12:18:37.881 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2024-02-07T12:18:37.881 DEBUG:teuthology.orchestra.run.smithi019:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-02-07T12:18:37.937 INFO:tasks.cephadm:Fetching mon keyring... 2024-02-07T12:18:37.937 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2024-02-07T12:18:37.937 DEBUG:teuthology.orchestra.run.smithi019:> sudo dd if=/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/mon.a/keyring of=/dev/stdout 2024-02-07T12:18:38.006 INFO:tasks.cephadm:Fetching pub ssh key... 2024-02-07T12:18:38.006 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2024-02-07T12:18:38.006 DEBUG:teuthology.orchestra.run.smithi019:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-02-07T12:18:38.061 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-02-07T12:18:38.061 DEBUG:teuthology.orchestra.run.smithi019:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC87Juc203zMsHCynIvpzKfCBp8d2JApSBh9ab8ICI3mPQ+QRE0xvluP14MQEKrrJA/pDRCMkPjCgrAacyjvR937OyBh+/QMP47lOhF+NJYSfFjQ1ssmq47CA/e5GmfR6SAxyaYWPMrlahb/RnMk6i8xT9hAEJmN/GOF0/KWwpC/yD+0ezo4p5NBGJhPDHN0DO2gqAbMt3G70Vcuhivimdb//e8GyKMru+nAA6SLwVO12BleGMDUcPl4c6oNK37V70LmMrKkwDdHxGgaGbT+YA0pB9R6aTTY/X7XA1Lxrg9sZlf6+ADJ4GtaKHiUpujLs0iykJGmUD9nzMJzz66BNRd5NV9zLRJQG9Q52Kcw01bAqN1odUBAM/BdhAKeoAO69kAlQfoR5KkjDObyZLWS8ybCzfZXWQkEMJkp98ZOY93CA3B1oDRK97udH6PLFwpSfHGTjY1yZWahNFBo/eRDUpmblHbfQnTMsNWsUaCqv8DaRZPaSXRXPX4srPzNGGU9yc= ceph-ced2783c-c5b2-11ee-95b6-87774f69a715' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-02-07T12:18:38.143 INFO:teuthology.orchestra.run.smithi019.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC87Juc203zMsHCynIvpzKfCBp8d2JApSBh9ab8ICI3mPQ+QRE0xvluP14MQEKrrJA/pDRCMkPjCgrAacyjvR937OyBh+/QMP47lOhF+NJYSfFjQ1ssmq47CA/e5GmfR6SAxyaYWPMrlahb/RnMk6i8xT9hAEJmN/GOF0/KWwpC/yD+0ezo4p5NBGJhPDHN0DO2gqAbMt3G70Vcuhivimdb//e8GyKMru+nAA6SLwVO12BleGMDUcPl4c6oNK37V70LmMrKkwDdHxGgaGbT+YA0pB9R6aTTY/X7XA1Lxrg9sZlf6+ADJ4GtaKHiUpujLs0iykJGmUD9nzMJzz66BNRd5NV9zLRJQG9Q52Kcw01bAqN1odUBAM/BdhAKeoAO69kAlQfoR5KkjDObyZLWS8ybCzfZXWQkEMJkp98ZOY93CA3B1oDRK97udH6PLFwpSfHGTjY1yZWahNFBo/eRDUpmblHbfQnTMsNWsUaCqv8DaRZPaSXRXPX4srPzNGGU9yc= ceph-ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:18:38.157 DEBUG:teuthology.orchestra.run.smithi172:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC87Juc203zMsHCynIvpzKfCBp8d2JApSBh9ab8ICI3mPQ+QRE0xvluP14MQEKrrJA/pDRCMkPjCgrAacyjvR937OyBh+/QMP47lOhF+NJYSfFjQ1ssmq47CA/e5GmfR6SAxyaYWPMrlahb/RnMk6i8xT9hAEJmN/GOF0/KWwpC/yD+0ezo4p5NBGJhPDHN0DO2gqAbMt3G70Vcuhivimdb//e8GyKMru+nAA6SLwVO12BleGMDUcPl4c6oNK37V70LmMrKkwDdHxGgaGbT+YA0pB9R6aTTY/X7XA1Lxrg9sZlf6+ADJ4GtaKHiUpujLs0iykJGmUD9nzMJzz66BNRd5NV9zLRJQG9Q52Kcw01bAqN1odUBAM/BdhAKeoAO69kAlQfoR5KkjDObyZLWS8ybCzfZXWQkEMJkp98ZOY93CA3B1oDRK97udH6PLFwpSfHGTjY1yZWahNFBo/eRDUpmblHbfQnTMsNWsUaCqv8DaRZPaSXRXPX4srPzNGGU9yc= ceph-ced2783c-c5b2-11ee-95b6-87774f69a715' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-02-07T12:18:38.204 INFO:teuthology.orchestra.run.smithi172.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC87Juc203zMsHCynIvpzKfCBp8d2JApSBh9ab8ICI3mPQ+QRE0xvluP14MQEKrrJA/pDRCMkPjCgrAacyjvR937OyBh+/QMP47lOhF+NJYSfFjQ1ssmq47CA/e5GmfR6SAxyaYWPMrlahb/RnMk6i8xT9hAEJmN/GOF0/KWwpC/yD+0ezo4p5NBGJhPDHN0DO2gqAbMt3G70Vcuhivimdb//e8GyKMru+nAA6SLwVO12BleGMDUcPl4c6oNK37V70LmMrKkwDdHxGgaGbT+YA0pB9R6aTTY/X7XA1Lxrg9sZlf6+ADJ4GtaKHiUpujLs0iykJGmUD9nzMJzz66BNRd5NV9zLRJQG9Q52Kcw01bAqN1odUBAM/BdhAKeoAO69kAlQfoR5KkjDObyZLWS8ybCzfZXWQkEMJkp98ZOY93CA3B1oDRK97udH6PLFwpSfHGTjY1yZWahNFBo/eRDUpmblHbfQnTMsNWsUaCqv8DaRZPaSXRXPX4srPzNGGU9yc= ceph-ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:18:38.220 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-02-07T12:18:38.525 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:38 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/4017590170' entity='client.admin' 2024-02-07T12:18:41.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:40 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:18:41.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:40 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi019", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:18:41.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:40 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:18:41.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:40 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:18:41.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:40 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:18:41.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:40 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2821803141' entity='client.admin' 2024-02-07T12:18:41.434 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-02-07T12:18:41.452 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-02-07T12:18:41.998 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:41 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:18:41.998 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:41 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:18:44.651 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi172 2024-02-07T12:18:44.651 DEBUG:teuthology.orchestra.run.smithi172:> set -ex 2024-02-07T12:18:44.652 DEBUG:teuthology.orchestra.run.smithi172:> dd of=/etc/ceph/ceph.conf 2024-02-07T12:18:44.670 DEBUG:teuthology.orchestra.run.smithi172:> set -ex 2024-02-07T12:18:44.670 DEBUG:teuthology.orchestra.run.smithi172:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:18:44.728 INFO:tasks.cephadm:Adding host smithi172 to orchestrator... 2024-02-07T12:18:44.729 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph orch host add smithi172 2024-02-07T12:18:45.232 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:44 smithi019 ceph-mon[24574]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:18:45.232 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:44 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:18:45.232 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:44 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:18:45.232 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:44 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:18:45.232 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:44 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:18:45.232 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:44 smithi019 ceph-mon[24574]: Updating smithi019:/etc/ceph/ceph.conf 2024-02-07T12:18:45.232 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:44 smithi019 ceph-mon[24574]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:18:45.232 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:44 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:18:45.232 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:44 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:18:47.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:46 smithi019 ceph-mon[24574]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi172", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:18:47.627 INFO:teuthology.orchestra.run.smithi019.stdout:Added host 'smithi172' with addr '172.21.15.172' 2024-02-07T12:18:48.193 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph orch host ls --format=json 2024-02-07T12:18:48.771 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:48 smithi019 ceph-mon[24574]: Deploying cephadm binary to smithi172 2024-02-07T12:18:48.771 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:48 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:18:48.772 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:48 smithi019 ceph-mon[24574]: Added host smithi172 2024-02-07T12:18:48.772 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:48 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:18:48.772 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:48 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:18:48.772 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:48 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:18:48.772 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:48 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:18:48.772 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:48 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:18:49.728 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:49 smithi019 ceph-mon[24574]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:18:49.728 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:49 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:18:50.021 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:18:50.021 INFO:teuthology.orchestra.run.smithi019.stdout:[{"addr": "172.21.15.19", "hostname": "smithi019", "labels": [], "status": ""}, {"addr": "172.21.15.172", "hostname": "smithi172", "labels": [], "status": ""}] 2024-02-07T12:18:50.601 INFO:tasks.cephadm:Setting crush tunables to default 2024-02-07T12:18:50.602 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph osd crush tunables default 2024-02-07T12:18:51.952 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:51 smithi019 ceph-mon[24574]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-07T12:18:51.952 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:51 smithi019 ceph-mon[24574]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:18:52.875 INFO:teuthology.orchestra.run.smithi019.stderr:adjusted tunables profile to default 2024-02-07T12:18:53.186 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:52 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:18:53.186 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:52 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:18:53.186 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:52 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi172", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:18:53.186 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:52 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:18:53.186 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:52 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2090489179' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-02-07T12:18:53.473 INFO:tasks.cephadm:Adding mon.a on smithi019 2024-02-07T12:18:53.473 INFO:tasks.cephadm:Adding mon.c on smithi019 2024-02-07T12:18:53.473 INFO:tasks.cephadm:Adding mon.b on smithi172 2024-02-07T12:18:53.473 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph orch apply mon '3;smithi019:172.21.15.19=a;smithi019:[v2:172.21.15.19:3301,v1:172.21.15.19:6790]=c;smithi172:172.21.15.172=b' 2024-02-07T12:18:54.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:53 smithi019 ceph-mon[24574]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:18:54.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:53 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2090489179' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-02-07T12:18:54.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:53 smithi019 ceph-mon[24574]: osdmap e4: 0 total, 0 up, 0 in 2024-02-07T12:18:55.583 INFO:teuthology.orchestra.run.smithi172.stdout:Scheduled mon update... 2024-02-07T12:18:56.154 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:55 smithi019 ceph-mon[24574]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:18:56.154 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:55 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:18:56.154 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:55 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:18:56.154 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:55 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:18:56.155 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:55 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:18:56.268 DEBUG:teuthology.orchestra.run.smithi019:mon.c> sudo journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mon.c.service 2024-02-07T12:18:56.270 DEBUG:teuthology.orchestra.run.smithi172:mon.b> sudo journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mon.b.service 2024-02-07T12:18:56.272 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-02-07T12:18:56.273 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph mon dump -f json 2024-02-07T12:18:57.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:56 smithi019 ceph-mon[24574]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi019:172.21.15.19=a;smithi019:[v2:172.21.15.19:3301,v1:172.21.15.19:6790]=c;smithi172:172.21.15.172=b", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:18:57.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:56 smithi019 ceph-mon[24574]: Saving service mon spec with placement smithi019:172.21.15.19=a;smithi019:[v2:172.21.15.19:3301,v1:172.21.15.19:6790]=c;smithi172:172.21.15.172=b;count:3 2024-02-07T12:18:57.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:56 smithi019 ceph-mon[24574]: Updating smithi172:/etc/ceph/ceph.conf 2024-02-07T12:18:57.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:56 smithi019 ceph-mon[24574]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:18:57.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:56 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:18:57.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:56 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:18:57.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:56 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:18:57.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:56 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:18:58.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:57 smithi019 ceph-mon[24574]: Deploying daemon mon.c on smithi019 2024-02-07T12:18:58.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:57 smithi019 ceph-mon[24574]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:18:58.210 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-02-07T12:18:58.210 INFO:teuthology.orchestra.run.smithi172.stdout:{"epoch":1,"fsid":"ced2783c-c5b2-11ee-95b6-87774f69a715","modified":"2024-02-07T12:17:35.503342Z","created":"2024-02-07T12:17:35.503342Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:3300","nonce":0},{"type":"v1","addr":"172.21.15.19:6789","nonce":0}]},"addr":"172.21.15.19:6789/0","public_addr":"172.21.15.19:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-02-07T12:18:58.212 INFO:teuthology.orchestra.run.smithi172.stderr:dumped monmap epoch 1 2024-02-07T12:18:59.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:18:58 smithi019 ceph-mon[24574]: from='client.? 172.21.15.172:0/3887620435' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:18:59.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:58 smithi019 podman[29247]: 2024-02-07 12:18:58.982493121 +0000 UTC m=+0.136834306 container create b9825d0db32d69a488d2490c8c4369dd4437443423031401a55d58061805812b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-c, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, release=754, version=8, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, ceph=True, vendor=Red Hat, Inc., architecture=x86_64, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, name=centos-stream, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-02-07T12:18:59.512 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 podman[29247]: 2024-02-07 12:18:59.193585539 +0000 UTC m=+0.347926699 container init b9825d0db32d69a488d2490c8c4369dd4437443423031401a55d58061805812b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-c, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., RELEASE=HEAD, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , io.openshift.expose-services=, vcs-type=git, GIT_BRANCH=HEAD, name=centos-stream, architecture=x86_64, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, version=8, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-02-07T12:18:59.512 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 podman[29247]: 2024-02-07 12:18:59.196959794 +0000 UTC m=+0.351300950 container start b9825d0db32d69a488d2490c8c4369dd4437443423031401a55d58061805812b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-c, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, version=8, ceph=True, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., release=754) 2024-02-07T12:18:59.513 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: set uid:gid to 167:167 (ceph:ceph) 2024-02-07T12:18:59.513 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-02-07T12:18:59.513 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: pidfile_write: ignore empty --pid-file 2024-02-07T12:18:59.513 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: load: jerasure load: lrc 2024-02-07T12:18:59.514 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: RocksDB version: 6.15.5 2024-02-07T12:18:59.514 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-02-07T12:18:59.514 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Compile date Apr 18 2022 2024-02-07T12:18:59.514 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: DB SUMMARY 2024-02-07T12:18:59.514 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: DB Session ID: 506I00OOGR0B14T8D8GZ 2024-02-07T12:18:59.514 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: CURRENT file: CURRENT 2024-02-07T12:18:59.514 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: IDENTITY file: IDENTITY 2024-02-07T12:18:59.514 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-02-07T12:18:59.514 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-02-07T12:18:59.514 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 511 ; 2024-02-07T12:18:59.514 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.error_if_exists: 0 2024-02-07T12:18:59.515 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.create_if_missing: 0 2024-02-07T12:18:59.515 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.paranoid_checks: 1 2024-02-07T12:18:59.515 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-02-07T12:18:59.515 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.env: 0x563d4c2c6860 2024-02-07T12:18:59.515 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.fs: Posix File System 2024-02-07T12:18:59.515 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.info_log: 0x563d4cdbddc0 2024-02-07T12:18:59.515 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.max_file_opening_threads: 16 2024-02-07T12:18:59.515 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.statistics: (nil) 2024-02-07T12:18:59.515 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.use_fsync: 0 2024-02-07T12:18:59.515 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.max_log_file_size: 0 2024-02-07T12:18:59.515 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-02-07T12:18:59.515 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.log_file_time_to_roll: 0 2024-02-07T12:18:59.515 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.keep_log_file_num: 1000 2024-02-07T12:18:59.515 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.recycle_log_file_num: 0 2024-02-07T12:18:59.515 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.allow_fallocate: 1 2024-02-07T12:18:59.516 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.allow_mmap_reads: 0 2024-02-07T12:18:59.516 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.allow_mmap_writes: 0 2024-02-07T12:18:59.516 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.use_direct_reads: 0 2024-02-07T12:18:59.516 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-02-07T12:18:59.516 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.create_missing_column_families: 0 2024-02-07T12:18:59.516 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.db_log_dir: 2024-02-07T12:18:59.516 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-02-07T12:18:59.516 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.table_cache_numshardbits: 6 2024-02-07T12:18:59.516 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.WAL_ttl_seconds: 0 2024-02-07T12:18:59.516 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.WAL_size_limit_MB: 0 2024-02-07T12:18:59.516 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-02-07T12:18:59.516 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-02-07T12:18:59.516 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.is_fd_close_on_exec: 1 2024-02-07T12:18:59.516 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.advise_random_on_open: 1 2024-02-07T12:18:59.516 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.db_write_buffer_size: 0 2024-02-07T12:18:59.517 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.write_buffer_manager: 0x563d4ceae2d0 2024-02-07T12:18:59.517 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-02-07T12:18:59.517 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-02-07T12:18:59.517 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-02-07T12:18:59.517 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.use_adaptive_mutex: 0 2024-02-07T12:18:59.517 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.rate_limiter: (nil) 2024-02-07T12:18:59.517 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-02-07T12:18:59.517 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.wal_recovery_mode: 2 2024-02-07T12:18:59.517 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.enable_thread_tracking: 0 2024-02-07T12:18:59.517 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.enable_pipelined_write: 0 2024-02-07T12:18:59.517 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.unordered_write: 0 2024-02-07T12:18:59.517 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-02-07T12:18:59.517 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-02-07T12:18:59.517 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-02-07T12:18:59.517 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-02-07T12:18:59.517 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.row_cache: None 2024-02-07T12:18:59.518 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.wal_filter: None 2024-02-07T12:18:59.518 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-02-07T12:18:59.518 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.allow_ingest_behind: 0 2024-02-07T12:18:59.518 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.preserve_deletes: 0 2024-02-07T12:18:59.518 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.two_write_queues: 0 2024-02-07T12:18:59.518 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.manual_wal_flush: 0 2024-02-07T12:18:59.518 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.atomic_flush: 0 2024-02-07T12:18:59.518 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-02-07T12:18:59.518 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.persist_stats_to_disk: 0 2024-02-07T12:18:59.518 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.write_dbid_to_manifest: 0 2024-02-07T12:18:59.518 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.log_readahead_size: 0 2024-02-07T12:18:59.518 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-02-07T12:18:59.518 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.best_efforts_recovery: 0 2024-02-07T12:18:59.518 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-02-07T12:18:59.518 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-02-07T12:18:59.518 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.allow_data_in_errors: 0 2024-02-07T12:18:59.519 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.db_host_id: __hostname__ 2024-02-07T12:18:59.519 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.max_background_jobs: 2 2024-02-07T12:18:59.519 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.max_background_compactions: -1 2024-02-07T12:18:59.519 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.max_subcompactions: 1 2024-02-07T12:18:59.519 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-02-07T12:18:59.519 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-02-07T12:18:59.519 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.delayed_write_rate : 16777216 2024-02-07T12:18:59.519 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.max_total_wal_size: 0 2024-02-07T12:18:59.519 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-02-07T12:18:59.519 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.stats_dump_period_sec: 600 2024-02-07T12:18:59.519 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.stats_persist_period_sec: 600 2024-02-07T12:18:59.519 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-02-07T12:18:59.519 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.max_open_files: -1 2024-02-07T12:18:59.519 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.bytes_per_sync: 0 2024-02-07T12:18:59.519 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.wal_bytes_per_sync: 0 2024-02-07T12:18:59.520 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.strict_bytes_per_sync: 0 2024-02-07T12:18:59.520 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.compaction_readahead_size: 0 2024-02-07T12:18:59.520 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.max_background_flushes: -1 2024-02-07T12:18:59.520 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Compression algorithms supported: 2024-02-07T12:18:59.520 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-02-07T12:18:59.520 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: kZSTD supported: 0 2024-02-07T12:18:59.520 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: kXpressCompression supported: 0 2024-02-07T12:18:59.520 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: kLZ4HCCompression supported: 1 2024-02-07T12:18:59.520 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: kLZ4Compression supported: 1 2024-02-07T12:18:59.520 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: kBZip2Compression supported: 0 2024-02-07T12:18:59.520 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: kZlibCompression supported: 1 2024-02-07T12:18:59.520 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: kSnappyCompression supported: 1 2024-02-07T12:18:59.520 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Fast CRC32 supported: Supported on x86 2024-02-07T12:18:59.520 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-02-07T12:18:59.520 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-02-07T12:18:59.521 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-02-07T12:18:59.521 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.merge_operator: 2024-02-07T12:18:59.521 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.compaction_filter: None 2024-02-07T12:18:59.521 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.compaction_filter_factory: None 2024-02-07T12:18:59.521 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.sst_partitioner_factory: None 2024-02-07T12:18:59.521 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.memtable_factory: SkipListFactory 2024-02-07T12:18:59.521 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.table_factory: BlockBasedTable 2024-02-07T12:18:59.521 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x563d4cd89d00) 2024-02-07T12:18:59.521 INFO:journalctl@ceph.mon.c.smithi019.stdout: cache_index_and_filter_blocks: 1 2024-02-07T12:18:59.521 INFO:journalctl@ceph.mon.c.smithi019.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-02-07T12:18:59.521 INFO:journalctl@ceph.mon.c.smithi019.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-02-07T12:18:59.521 INFO:journalctl@ceph.mon.c.smithi019.stdout: pin_top_level_index_and_filter: 1 2024-02-07T12:18:59.521 INFO:journalctl@ceph.mon.c.smithi019.stdout: index_type: 0 2024-02-07T12:18:59.521 INFO:journalctl@ceph.mon.c.smithi019.stdout: data_block_index_type: 0 2024-02-07T12:18:59.521 INFO:journalctl@ceph.mon.c.smithi019.stdout: index_shortening: 1 2024-02-07T12:18:59.521 INFO:journalctl@ceph.mon.c.smithi019.stdout: data_block_hash_table_util_ratio: 0.750000 2024-02-07T12:18:59.522 INFO:journalctl@ceph.mon.c.smithi019.stdout: hash_index_allow_collision: 1 2024-02-07T12:18:59.522 INFO:journalctl@ceph.mon.c.smithi019.stdout: checksum: 1 2024-02-07T12:18:59.522 INFO:journalctl@ceph.mon.c.smithi019.stdout: no_block_cache: 0 2024-02-07T12:18:59.522 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_cache: 0x563d4cdf4170 2024-02-07T12:18:59.522 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_cache_name: BinnedLRUCache 2024-02-07T12:18:59.522 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_cache_options: 2024-02-07T12:18:59.522 INFO:journalctl@ceph.mon.c.smithi019.stdout: capacity : 536870912 2024-02-07T12:18:59.522 INFO:journalctl@ceph.mon.c.smithi019.stdout: num_shard_bits : 4 2024-02-07T12:18:59.522 INFO:journalctl@ceph.mon.c.smithi019.stdout: strict_capacity_limit : 0 2024-02-07T12:18:59.522 INFO:journalctl@ceph.mon.c.smithi019.stdout: high_pri_pool_ratio: 0.000 2024-02-07T12:18:59.522 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_cache_compressed: (nil) 2024-02-07T12:18:59.522 INFO:journalctl@ceph.mon.c.smithi019.stdout: persistent_cache: (nil) 2024-02-07T12:18:59.522 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_size: 4096 2024-02-07T12:18:59.522 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_size_deviation: 10 2024-02-07T12:18:59.522 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_restart_interval: 16 2024-02-07T12:18:59.523 INFO:journalctl@ceph.mon.c.smithi019.stdout: index_block_restart_interval: 1 2024-02-07T12:18:59.523 INFO:journalctl@ceph.mon.c.smithi019.stdout: metadata_block_size: 4096 2024-02-07T12:18:59.523 INFO:journalctl@ceph.mon.c.smithi019.stdout: partition_filters: 0 2024-02-07T12:18:59.523 INFO:journalctl@ceph.mon.c.smithi019.stdout: use_delta_encoding: 1 2024-02-07T12:18:59.523 INFO:journalctl@ceph.mon.c.smithi019.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-02-07T12:18:59.523 INFO:journalctl@ceph.mon.c.smithi019.stdout: whole_key_filtering: 1 2024-02-07T12:18:59.523 INFO:journalctl@ceph.mon.c.smithi019.stdout: verify_compression: 0 2024-02-07T12:18:59.523 INFO:journalctl@ceph.mon.c.smithi019.stdout: read_amp_bytes_per_bit: 0 2024-02-07T12:18:59.523 INFO:journalctl@ceph.mon.c.smithi019.stdout: format_version: 4 2024-02-07T12:18:59.523 INFO:journalctl@ceph.mon.c.smithi019.stdout: enable_index_compression: 1 2024-02-07T12:18:59.523 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_align: 0 2024-02-07T12:18:59.523 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.write_buffer_size: 33554432 2024-02-07T12:18:59.523 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.max_write_buffer_number: 2 2024-02-07T12:18:59.523 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.compression: NoCompression 2024-02-07T12:18:59.523 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.bottommost_compression: Disabled 2024-02-07T12:18:59.523 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.prefix_extractor: nullptr 2024-02-07T12:18:59.524 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-02-07T12:18:59.524 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.num_levels: 7 2024-02-07T12:18:59.524 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-02-07T12:18:59.524 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-02-07T12:18:59.524 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-02-07T12:18:59.524 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-02-07T12:18:59.524 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-02-07T12:18:59.524 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-02-07T12:18:59.524 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-02-07T12:18:59.524 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-02-07T12:18:59.524 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-02-07T12:18:59.524 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-02-07T12:18:59.524 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.compression_opts.window_bits: -14 2024-02-07T12:18:59.524 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.compression_opts.level: 32767 2024-02-07T12:18:59.524 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.compression_opts.strategy: 0 2024-02-07T12:18:59.525 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-02-07T12:18:59.525 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-02-07T12:18:59.525 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-02-07T12:18:59.525 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.compression_opts.enabled: false 2024-02-07T12:18:59.526 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-02-07T12:18:59.526 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-02-07T12:18:59.526 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-02-07T12:18:59.526 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.target_file_size_base: 67108864 2024-02-07T12:18:59.526 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.target_file_size_multiplier: 1 2024-02-07T12:18:59.526 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-02-07T12:18:59.526 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-02-07T12:18:59.526 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-02-07T12:18:59.526 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-02-07T12:18:59.526 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-02-07T12:18:59.526 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-02-07T12:18:59.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-02-07T12:18:59.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-02-07T12:18:59.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-02-07T12:18:59.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-02-07T12:18:59.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-02-07T12:18:59.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-02-07T12:18:59.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.arena_block_size: 4194304 2024-02-07T12:18:59.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-02-07T12:18:59.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-02-07T12:18:59.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-02-07T12:18:59.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.disable_auto_compactions: 0 2024-02-07T12:18:59.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-02-07T12:18:59.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-02-07T12:18:59.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-02-07T12:18:59.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-02-07T12:18:59.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-02-07T12:18:59.528 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-02-07T12:18:59.528 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-02-07T12:18:59.528 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-02-07T12:18:59.528 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-02-07T12:18:59.528 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-02-07T12:18:59.528 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.table_properties_collectors: 2024-02-07T12:18:59.528 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.inplace_update_support: 0 2024-02-07T12:18:59.528 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.inplace_update_num_locks: 10000 2024-02-07T12:18:59.528 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-02-07T12:18:59.528 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-02-07T12:18:59.528 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.memtable_huge_page_size: 0 2024-02-07T12:18:59.528 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.bloom_locality: 0 2024-02-07T12:18:59.528 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.max_successive_merges: 0 2024-02-07T12:18:59.528 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.optimize_filters_for_hits: 0 2024-02-07T12:18:59.528 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.paranoid_file_checks: 0 2024-02-07T12:18:59.529 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.force_consistency_checks: 1 2024-02-07T12:18:59.529 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.report_bg_io_stats: 0 2024-02-07T12:18:59.529 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.ttl: 2592000 2024-02-07T12:18:59.529 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.periodic_compaction_seconds: 0 2024-02-07T12:18:59.529 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.enable_blob_files: false 2024-02-07T12:18:59.529 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.min_blob_size: 0 2024-02-07T12:18:59.529 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.blob_file_size: 268435456 2024-02-07T12:18:59.529 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.blob_compression_type: NoCompression 2024-02-07T12:18:59.529 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.enable_blob_garbage_collection: false 2024-02-07T12:18:59.529 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-02-07T12:18:59.529 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-02-07T12:18:59.529 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-02-07T12:18:59.529 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-02-07T12:18:59.529 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707308339222919, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-02-07T12:18:59.529 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-02-07T12:18:59.529 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-02-07T12:18:59.529 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707308339223223, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1707308339, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "14ff0c38-ce13-4ecb-a9dc-f7eb8f30f00b", "db_session_id": "506I00OOGR0B14T8D8GZ"}} 2024-02-07T12:18:59.530 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-02-07T12:18:59.530 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707308339223555, "job": 1, "event": "recovery_finished"} 2024-02-07T12:18:59.530 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-02-07T12:18:59.530 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x563d4cddaa80 2024-02-07T12:18:59.530 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: DB pointer 0x563d4cdea000 2024-02-07T12:18:59.530 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-02-07T12:18:59.530 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-02-07T12:18:59.530 INFO:journalctl@ceph.mon.c.smithi019.stdout: ** DB Stats ** 2024-02-07T12:18:59.530 INFO:journalctl@ceph.mon.c.smithi019.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-02-07T12:18:59.530 INFO:journalctl@ceph.mon.c.smithi019.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-02-07T12:18:59.530 INFO:journalctl@ceph.mon.c.smithi019.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-02-07T12:18:59.530 INFO:journalctl@ceph.mon.c.smithi019.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-07T12:18:59.531 INFO:journalctl@ceph.mon.c.smithi019.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-02-07T12:18:59.531 INFO:journalctl@ceph.mon.c.smithi019.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-02-07T12:18:59.531 INFO:journalctl@ceph.mon.c.smithi019.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-07T12:18:59.531 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2024-02-07T12:18:59.531 INFO:journalctl@ceph.mon.c.smithi019.stdout: ** Compaction Stats [default] ** 2024-02-07T12:18:59.531 INFO:journalctl@ceph.mon.c.smithi019.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-07T12:18:59.531 INFO:journalctl@ceph.mon.c.smithi019.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-07T12:18:59.531 INFO:journalctl@ceph.mon.c.smithi019.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-02-07T12:18:59.531 INFO:journalctl@ceph.mon.c.smithi019.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-02-07T12:18:59.531 INFO:journalctl@ceph.mon.c.smithi019.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-02-07T12:18:59.531 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2024-02-07T12:18:59.531 INFO:journalctl@ceph.mon.c.smithi019.stdout: ** Compaction Stats [default] ** 2024-02-07T12:18:59.531 INFO:journalctl@ceph.mon.c.smithi019.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-07T12:18:59.531 INFO:journalctl@ceph.mon.c.smithi019.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-07T12:18:59.531 INFO:journalctl@ceph.mon.c.smithi019.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-02-07T12:18:59.531 INFO:journalctl@ceph.mon.c.smithi019.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-02-07T12:18:59.532 INFO:journalctl@ceph.mon.c.smithi019.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-02-07T12:18:59.532 INFO:journalctl@ceph.mon.c.smithi019.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-02-07T12:18:59.532 INFO:journalctl@ceph.mon.c.smithi019.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-02-07T12:18:59.532 INFO:journalctl@ceph.mon.c.smithi019.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-02-07T12:18:59.532 INFO:journalctl@ceph.mon.c.smithi019.stdout: AddFile(Keys): cumulative 0, interval 0 2024-02-07T12:18:59.532 INFO:journalctl@ceph.mon.c.smithi019.stdout: Cumulative compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:18:59.532 INFO:journalctl@ceph.mon.c.smithi019.stdout: Interval compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:18:59.532 INFO:journalctl@ceph.mon.c.smithi019.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-02-07T12:18:59.532 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2024-02-07T12:18:59.532 INFO:journalctl@ceph.mon.c.smithi019.stdout: ** File Read Latency Histogram By Level [default] ** 2024-02-07T12:18:59.533 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2024-02-07T12:18:59.533 INFO:journalctl@ceph.mon.c.smithi019.stdout: ** Compaction Stats [default] ** 2024-02-07T12:18:59.533 INFO:journalctl@ceph.mon.c.smithi019.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-07T12:18:59.533 INFO:journalctl@ceph.mon.c.smithi019.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-07T12:18:59.533 INFO:journalctl@ceph.mon.c.smithi019.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-02-07T12:18:59.533 INFO:journalctl@ceph.mon.c.smithi019.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-02-07T12:18:59.533 INFO:journalctl@ceph.mon.c.smithi019.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-02-07T12:18:59.533 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2024-02-07T12:18:59.533 INFO:journalctl@ceph.mon.c.smithi019.stdout: ** Compaction Stats [default] ** 2024-02-07T12:18:59.534 INFO:journalctl@ceph.mon.c.smithi019.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-07T12:18:59.534 INFO:journalctl@ceph.mon.c.smithi019.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-07T12:18:59.534 INFO:journalctl@ceph.mon.c.smithi019.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-02-07T12:18:59.535 INFO:journalctl@ceph.mon.c.smithi019.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-02-07T12:18:59.535 INFO:journalctl@ceph.mon.c.smithi019.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-02-07T12:18:59.535 INFO:journalctl@ceph.mon.c.smithi019.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-02-07T12:18:59.535 INFO:journalctl@ceph.mon.c.smithi019.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-02-07T12:18:59.535 INFO:journalctl@ceph.mon.c.smithi019.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-02-07T12:18:59.535 INFO:journalctl@ceph.mon.c.smithi019.stdout: AddFile(Keys): cumulative 0, interval 0 2024-02-07T12:18:59.536 INFO:journalctl@ceph.mon.c.smithi019.stdout: Cumulative compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:18:59.536 INFO:journalctl@ceph.mon.c.smithi019.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:18:59.536 INFO:journalctl@ceph.mon.c.smithi019.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-02-07T12:18:59.536 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2024-02-07T12:18:59.536 INFO:journalctl@ceph.mon.c.smithi019.stdout: ** File Read Latency Histogram By Level [default] ** 2024-02-07T12:18:59.536 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-02-07T12:18:59.536 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: using public_addrv [v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0] 2024-02-07T12:18:59.537 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: starting mon.c rank -1 at public addrs [v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0] at bind addrs [v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:18:59.537 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: mon.c@-1(???) e0 preinit fsid ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:18:59.537 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: mon.c@-1(synchronizing).mds e1 new map 2024-02-07T12:18:59.537 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: mon.c@-1(synchronizing).mds e1 print_map 2024-02-07T12:18:59.537 INFO:journalctl@ceph.mon.c.smithi019.stdout: e1 2024-02-07T12:18:59.537 INFO:journalctl@ceph.mon.c.smithi019.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-02-07T12:18:59.537 INFO:journalctl@ceph.mon.c.smithi019.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-02-07T12:18:59.538 INFO:journalctl@ceph.mon.c.smithi019.stdout: legacy client fscid: -1 2024-02-07T12:18:59.538 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2024-02-07T12:18:59.538 INFO:journalctl@ceph.mon.c.smithi019.stdout: No filesystems configured 2024-02-07T12:18:59.538 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-02-07T12:18:59.538 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-02-07T12:18:59.538 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-02-07T12:18:59.538 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-02-07T12:18:59.538 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-02-07T12:18:59.539 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-02-07T12:18:59.539 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-02-07T12:18:59.539 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-02-07T12:18:59.539 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-02-07T12:18:59.539 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-02-07T12:18:59.539 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: Deploying daemon mon.c on smithi019 2024-02-07T12:18:59.539 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:18:59.539 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: from='client.? 172.21.15.172:0/3887620435' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:18:59.539 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-02-07T12:18:59.540 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 ceph-mon[29267]: mon.c@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-02-07T12:18:59.540 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 bash[29247]: b9825d0db32d69a488d2490c8c4369dd4437443423031401a55d58061805812b 2024-02-07T12:18:59.540 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:18:59 smithi019 systemd[1]: Started Ceph mon.c for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:18:59.739 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-02-07T12:18:59.740 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph mon dump -f json 2024-02-07T12:19:04.590 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-02-07T12:19:04.590 INFO:teuthology.orchestra.run.smithi172.stdout:{"epoch":2,"fsid":"ced2783c-c5b2-11ee-95b6-87774f69a715","modified":"2024-02-07T12:18:59.241654Z","created":"2024-02-07T12:17:35.503342Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:3300","nonce":0},{"type":"v1","addr":"172.21.15.19:6789","nonce":0}]},"addr":"172.21.15.19:6789/0","public_addr":"172.21.15.19:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:3301","nonce":0},{"type":"v1","addr":"172.21.15.19:6790","nonce":0}]},"addr":"172.21.15.19:6790/0","public_addr":"172.21.15.19:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-02-07T12:19:04.593 INFO:teuthology.orchestra.run.smithi172.stderr:dumped monmap epoch 2 2024-02-07T12:19:04.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:19:04.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[24574]: mon.a calling monitor election 2024-02-07T12:19:04.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:19:04.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:19:04.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[24574]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:04.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:19:04.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[24574]: mon.c calling monitor election 2024-02-07T12:19:04.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:19:04.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[24574]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:04.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:19:04.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:19:04.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[24574]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-02-07T12:19:04.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[24574]: monmap e2: 2 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} 2024-02-07T12:19:04.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[24574]: fsmap 2024-02-07T12:19:04.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[24574]: osdmap e4: 0 total, 0 up, 0 in 2024-02-07T12:19:04.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[24574]: mgrmap e13: y(active, since 36s) 2024-02-07T12:19:04.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[24574]: overall HEALTH_OK 2024-02-07T12:19:04.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:04.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:19:04.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:04.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:19:04.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[29267]: mon.a calling monitor election 2024-02-07T12:19:04.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:19:04.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:19:04.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[29267]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:04.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:19:04.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[29267]: mon.c calling monitor election 2024-02-07T12:19:04.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:19:04.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[29267]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:04.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:19:04.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:19:04.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[29267]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-02-07T12:19:04.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[29267]: monmap e2: 2 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} 2024-02-07T12:19:04.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[29267]: fsmap 2024-02-07T12:19:04.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[29267]: osdmap e4: 0 total, 0 up, 0 in 2024-02-07T12:19:04.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[29267]: mgrmap e13: y(active, since 36s) 2024-02-07T12:19:04.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[29267]: overall HEALTH_OK 2024-02-07T12:19:04.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:04.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:19:04.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:04 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:05.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:05 smithi019 ceph-mon[24574]: Deploying daemon mon.b on smithi172 2024-02-07T12:19:05.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:05 smithi019 ceph-mon[24574]: from='client.? 172.21.15.172:0/129446763' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:19:05.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:05 smithi019 ceph-mon[24574]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:05.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:05 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:19:05.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:05 smithi019 ceph-mon[29267]: Deploying daemon mon.b on smithi172 2024-02-07T12:19:05.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:05 smithi019 ceph-mon[29267]: from='client.? 172.21.15.172:0/129446763' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:19:05.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:05 smithi019 ceph-mon[29267]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:05.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:05 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:19:06.664 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-02-07T12:19:06.664 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph mon dump -f json 2024-02-07T12:19:08.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:07 smithi019 ceph-mon[24574]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:08.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:07 smithi019 ceph-mon[29267]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:08.881 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-02-07T12:19:08.881 INFO:teuthology.orchestra.run.smithi172.stdout:{"epoch":2,"fsid":"ced2783c-c5b2-11ee-95b6-87774f69a715","modified":"2024-02-07T12:18:59.241654Z","created":"2024-02-07T12:17:35.503342Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:3300","nonce":0},{"type":"v1","addr":"172.21.15.19:6789","nonce":0}]},"addr":"172.21.15.19:6789/0","public_addr":"172.21.15.19:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:3301","nonce":0},{"type":"v1","addr":"172.21.15.19:6790","nonce":0}]},"addr":"172.21.15.19:6790/0","public_addr":"172.21.15.19:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-02-07T12:19:08.881 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:08 smithi172 podman[25424]: 2024-02-07 12:19:08.795022181 +0000 UTC m=+0.125746902 container create 7d62c7c4d075b789ea2a026b6848c89c9273a14590b88e7050e7e1d8a37ff299 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-b, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, distribution-scope=public, ceph=True, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, build-date=2022-05-03T08:36:31.336870, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc.) 2024-02-07T12:19:08.882 INFO:teuthology.orchestra.run.smithi172.stderr:dumped monmap epoch 2 2024-02-07T12:19:09.577 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 podman[25424]: 2024-02-07 12:19:09.181507825 +0000 UTC m=+0.512232547 container init 7d62c7c4d075b789ea2a026b6848c89c9273a14590b88e7050e7e1d8a37ff299 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-b, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, version=8, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc.) 2024-02-07T12:19:09.577 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 podman[25424]: 2024-02-07 12:19:09.185002119 +0000 UTC m=+0.515726844 container start 7d62c7c4d075b789ea2a026b6848c89c9273a14590b88e7050e7e1d8a37ff299 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-b, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, RELEASE=HEAD, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, io.buildah.version=1.19.8, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, ceph=True, build-date=2022-05-03T08:36:31.336870, release=754, distribution-scope=public) 2024-02-07T12:19:09.577 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: set uid:gid to 167:167 (ceph:ceph) 2024-02-07T12:19:09.577 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-02-07T12:19:09.577 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: pidfile_write: ignore empty --pid-file 2024-02-07T12:19:09.577 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: load: jerasure load: lrc 2024-02-07T12:19:09.578 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: RocksDB version: 6.15.5 2024-02-07T12:19:09.578 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-02-07T12:19:09.578 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Compile date Apr 18 2022 2024-02-07T12:19:09.578 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: DB SUMMARY 2024-02-07T12:19:09.578 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: DB Session ID: Z94JZNK67WPI661Y46ZP 2024-02-07T12:19:09.578 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: CURRENT file: CURRENT 2024-02-07T12:19:09.578 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: IDENTITY file: IDENTITY 2024-02-07T12:19:09.578 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-02-07T12:19:09.578 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-02-07T12:19:09.578 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 636 ; 2024-02-07T12:19:09.579 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.error_if_exists: 0 2024-02-07T12:19:09.579 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.create_if_missing: 0 2024-02-07T12:19:09.579 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.paranoid_checks: 1 2024-02-07T12:19:09.579 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-02-07T12:19:09.579 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.env: 0x555b7f8c6860 2024-02-07T12:19:09.579 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.fs: Posix File System 2024-02-07T12:19:09.579 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.info_log: 0x555b814dbdc0 2024-02-07T12:19:09.579 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.max_file_opening_threads: 16 2024-02-07T12:19:09.579 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.statistics: (nil) 2024-02-07T12:19:09.579 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.use_fsync: 0 2024-02-07T12:19:09.579 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.max_log_file_size: 0 2024-02-07T12:19:09.579 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-02-07T12:19:09.579 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.log_file_time_to_roll: 0 2024-02-07T12:19:09.579 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.keep_log_file_num: 1000 2024-02-07T12:19:09.580 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.recycle_log_file_num: 0 2024-02-07T12:19:09.580 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.allow_fallocate: 1 2024-02-07T12:19:09.580 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.allow_mmap_reads: 0 2024-02-07T12:19:09.580 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.allow_mmap_writes: 0 2024-02-07T12:19:09.580 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.use_direct_reads: 0 2024-02-07T12:19:09.580 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-02-07T12:19:09.580 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.create_missing_column_families: 0 2024-02-07T12:19:09.580 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.db_log_dir: 2024-02-07T12:19:09.580 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-02-07T12:19:09.580 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.table_cache_numshardbits: 6 2024-02-07T12:19:09.580 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.WAL_ttl_seconds: 0 2024-02-07T12:19:09.580 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.WAL_size_limit_MB: 0 2024-02-07T12:19:09.581 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-02-07T12:19:09.581 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-02-07T12:19:09.581 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.is_fd_close_on_exec: 1 2024-02-07T12:19:09.582 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.advise_random_on_open: 1 2024-02-07T12:19:09.582 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.db_write_buffer_size: 0 2024-02-07T12:19:09.582 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.write_buffer_manager: 0x555b815cc240 2024-02-07T12:19:09.582 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-02-07T12:19:09.582 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-02-07T12:19:09.582 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-02-07T12:19:09.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.use_adaptive_mutex: 0 2024-02-07T12:19:09.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.rate_limiter: (nil) 2024-02-07T12:19:09.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-02-07T12:19:09.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.wal_recovery_mode: 2 2024-02-07T12:19:09.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.enable_thread_tracking: 0 2024-02-07T12:19:09.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.enable_pipelined_write: 0 2024-02-07T12:19:09.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.unordered_write: 0 2024-02-07T12:19:09.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-02-07T12:19:09.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-02-07T12:19:09.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-02-07T12:19:09.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-02-07T12:19:09.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.row_cache: None 2024-02-07T12:19:09.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.wal_filter: None 2024-02-07T12:19:09.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-02-07T12:19:09.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.allow_ingest_behind: 0 2024-02-07T12:19:09.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.preserve_deletes: 0 2024-02-07T12:19:09.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.two_write_queues: 0 2024-02-07T12:19:09.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.manual_wal_flush: 0 2024-02-07T12:19:09.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.atomic_flush: 0 2024-02-07T12:19:09.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-02-07T12:19:09.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.persist_stats_to_disk: 0 2024-02-07T12:19:09.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.write_dbid_to_manifest: 0 2024-02-07T12:19:09.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.log_readahead_size: 0 2024-02-07T12:19:09.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-02-07T12:19:09.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.best_efforts_recovery: 0 2024-02-07T12:19:09.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-02-07T12:19:09.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-02-07T12:19:09.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.allow_data_in_errors: 0 2024-02-07T12:19:09.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.db_host_id: __hostname__ 2024-02-07T12:19:09.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.max_background_jobs: 2 2024-02-07T12:19:09.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.max_background_compactions: -1 2024-02-07T12:19:09.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.max_subcompactions: 1 2024-02-07T12:19:09.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-02-07T12:19:09.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-02-07T12:19:09.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.delayed_write_rate : 16777216 2024-02-07T12:19:09.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.max_total_wal_size: 0 2024-02-07T12:19:09.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-02-07T12:19:09.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.stats_dump_period_sec: 600 2024-02-07T12:19:09.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.stats_persist_period_sec: 600 2024-02-07T12:19:09.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-02-07T12:19:09.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.max_open_files: -1 2024-02-07T12:19:09.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.bytes_per_sync: 0 2024-02-07T12:19:09.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.wal_bytes_per_sync: 0 2024-02-07T12:19:09.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.strict_bytes_per_sync: 0 2024-02-07T12:19:09.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.compaction_readahead_size: 0 2024-02-07T12:19:09.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.max_background_flushes: -1 2024-02-07T12:19:09.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Compression algorithms supported: 2024-02-07T12:19:09.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-02-07T12:19:09.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: kZSTD supported: 0 2024-02-07T12:19:09.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: kXpressCompression supported: 0 2024-02-07T12:19:09.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: kLZ4HCCompression supported: 1 2024-02-07T12:19:09.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: kLZ4Compression supported: 1 2024-02-07T12:19:09.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: kBZip2Compression supported: 0 2024-02-07T12:19:09.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: kZlibCompression supported: 1 2024-02-07T12:19:09.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: kSnappyCompression supported: 1 2024-02-07T12:19:09.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Fast CRC32 supported: Supported on x86 2024-02-07T12:19:09.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-02-07T12:19:09.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-02-07T12:19:09.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-02-07T12:19:09.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.merge_operator: 2024-02-07T12:19:09.587 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.compaction_filter: None 2024-02-07T12:19:09.587 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.compaction_filter_factory: None 2024-02-07T12:19:09.587 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.sst_partitioner_factory: None 2024-02-07T12:19:09.587 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.memtable_factory: SkipListFactory 2024-02-07T12:19:09.587 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.table_factory: BlockBasedTable 2024-02-07T12:19:09.587 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x555b814a7d00) 2024-02-07T12:19:09.587 INFO:journalctl@ceph.mon.b.smithi172.stdout: cache_index_and_filter_blocks: 1 2024-02-07T12:19:09.587 INFO:journalctl@ceph.mon.b.smithi172.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-02-07T12:19:09.587 INFO:journalctl@ceph.mon.b.smithi172.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-02-07T12:19:09.587 INFO:journalctl@ceph.mon.b.smithi172.stdout: pin_top_level_index_and_filter: 1 2024-02-07T12:19:09.587 INFO:journalctl@ceph.mon.b.smithi172.stdout: index_type: 0 2024-02-07T12:19:09.587 INFO:journalctl@ceph.mon.b.smithi172.stdout: data_block_index_type: 0 2024-02-07T12:19:09.587 INFO:journalctl@ceph.mon.b.smithi172.stdout: index_shortening: 1 2024-02-07T12:19:09.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: data_block_hash_table_util_ratio: 0.750000 2024-02-07T12:19:09.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: hash_index_allow_collision: 1 2024-02-07T12:19:09.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: checksum: 1 2024-02-07T12:19:09.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: no_block_cache: 0 2024-02-07T12:19:09.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_cache: 0x555b81512170 2024-02-07T12:19:09.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_cache_name: BinnedLRUCache 2024-02-07T12:19:09.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_cache_options: 2024-02-07T12:19:09.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: capacity : 536870912 2024-02-07T12:19:09.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: num_shard_bits : 4 2024-02-07T12:19:09.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: strict_capacity_limit : 0 2024-02-07T12:19:09.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: high_pri_pool_ratio: 0.000 2024-02-07T12:19:09.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_cache_compressed: (nil) 2024-02-07T12:19:09.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: persistent_cache: (nil) 2024-02-07T12:19:09.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_size: 4096 2024-02-07T12:19:09.589 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_size_deviation: 10 2024-02-07T12:19:09.589 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_restart_interval: 16 2024-02-07T12:19:09.589 INFO:journalctl@ceph.mon.b.smithi172.stdout: index_block_restart_interval: 1 2024-02-07T12:19:09.589 INFO:journalctl@ceph.mon.b.smithi172.stdout: metadata_block_size: 4096 2024-02-07T12:19:09.589 INFO:journalctl@ceph.mon.b.smithi172.stdout: partition_filters: 0 2024-02-07T12:19:09.589 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_delta_encoding: 1 2024-02-07T12:19:09.589 INFO:journalctl@ceph.mon.b.smithi172.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-02-07T12:19:09.589 INFO:journalctl@ceph.mon.b.smithi172.stdout: whole_key_filtering: 1 2024-02-07T12:19:09.589 INFO:journalctl@ceph.mon.b.smithi172.stdout: verify_compression: 0 2024-02-07T12:19:09.589 INFO:journalctl@ceph.mon.b.smithi172.stdout: read_amp_bytes_per_bit: 0 2024-02-07T12:19:09.589 INFO:journalctl@ceph.mon.b.smithi172.stdout: format_version: 4 2024-02-07T12:19:09.589 INFO:journalctl@ceph.mon.b.smithi172.stdout: enable_index_compression: 1 2024-02-07T12:19:09.589 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_align: 0 2024-02-07T12:19:09.589 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.write_buffer_size: 33554432 2024-02-07T12:19:09.589 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.max_write_buffer_number: 2 2024-02-07T12:19:09.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.compression: NoCompression 2024-02-07T12:19:09.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.bottommost_compression: Disabled 2024-02-07T12:19:09.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.prefix_extractor: nullptr 2024-02-07T12:19:09.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-02-07T12:19:09.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.num_levels: 7 2024-02-07T12:19:09.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-02-07T12:19:09.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-02-07T12:19:09.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-02-07T12:19:09.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-02-07T12:19:09.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-02-07T12:19:09.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-02-07T12:19:09.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-02-07T12:19:09.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-02-07T12:19:09.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-02-07T12:19:09.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-02-07T12:19:09.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.compression_opts.window_bits: -14 2024-02-07T12:19:09.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.compression_opts.level: 32767 2024-02-07T12:19:09.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.compression_opts.strategy: 0 2024-02-07T12:19:09.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-02-07T12:19:09.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-02-07T12:19:09.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-02-07T12:19:09.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.compression_opts.enabled: false 2024-02-07T12:19:09.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-02-07T12:19:09.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-02-07T12:19:09.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-02-07T12:19:09.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.target_file_size_base: 67108864 2024-02-07T12:19:09.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.target_file_size_multiplier: 1 2024-02-07T12:19:09.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-02-07T12:19:09.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-02-07T12:19:09.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-02-07T12:19:09.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-02-07T12:19:09.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-02-07T12:19:09.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-02-07T12:19:09.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-02-07T12:19:09.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-02-07T12:19:09.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-02-07T12:19:09.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-02-07T12:19:09.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-02-07T12:19:09.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-02-07T12:19:09.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.arena_block_size: 4194304 2024-02-07T12:19:09.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-02-07T12:19:09.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-02-07T12:19:09.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-02-07T12:19:09.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.disable_auto_compactions: 0 2024-02-07T12:19:09.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-02-07T12:19:09.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-02-07T12:19:09.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-02-07T12:19:09.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-02-07T12:19:09.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-02-07T12:19:09.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-02-07T12:19:09.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-02-07T12:19:09.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-02-07T12:19:09.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-02-07T12:19:09.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-02-07T12:19:09.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.table_properties_collectors: 2024-02-07T12:19:09.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.inplace_update_support: 0 2024-02-07T12:19:09.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.inplace_update_num_locks: 10000 2024-02-07T12:19:09.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-02-07T12:19:09.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-02-07T12:19:09.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.memtable_huge_page_size: 0 2024-02-07T12:19:09.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.bloom_locality: 0 2024-02-07T12:19:09.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.max_successive_merges: 0 2024-02-07T12:19:09.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.optimize_filters_for_hits: 0 2024-02-07T12:19:09.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.paranoid_file_checks: 0 2024-02-07T12:19:09.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.force_consistency_checks: 1 2024-02-07T12:19:09.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.report_bg_io_stats: 0 2024-02-07T12:19:09.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.ttl: 2592000 2024-02-07T12:19:09.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.periodic_compaction_seconds: 0 2024-02-07T12:19:09.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.enable_blob_files: false 2024-02-07T12:19:09.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.min_blob_size: 0 2024-02-07T12:19:09.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.blob_file_size: 268435456 2024-02-07T12:19:09.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.blob_compression_type: NoCompression 2024-02-07T12:19:09.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.enable_blob_garbage_collection: false 2024-02-07T12:19:09.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-02-07T12:19:09.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-02-07T12:19:09.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-02-07T12:19:09.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-02-07T12:19:09.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707308349211525, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-02-07T12:19:09.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-02-07T12:19:09.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-02-07T12:19:09.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707308349211827, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1707308349, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "4776c340-4345-44aa-954c-47935994e255", "db_session_id": "Z94JZNK67WPI661Y46ZP"}} 2024-02-07T12:19:09.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-02-07T12:19:09.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707308349212131, "job": 1, "event": "recovery_finished"} 2024-02-07T12:19:09.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-02-07T12:19:09.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x555b814f8a80 2024-02-07T12:19:09.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: DB pointer 0x555b81508000 2024-02-07T12:19:09.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-02-07T12:19:09.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-02-07T12:19:09.596 INFO:journalctl@ceph.mon.b.smithi172.stdout: ** DB Stats ** 2024-02-07T12:19:09.596 INFO:journalctl@ceph.mon.b.smithi172.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-02-07T12:19:09.596 INFO:journalctl@ceph.mon.b.smithi172.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-02-07T12:19:09.596 INFO:journalctl@ceph.mon.b.smithi172.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-02-07T12:19:09.596 INFO:journalctl@ceph.mon.b.smithi172.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-07T12:19:09.596 INFO:journalctl@ceph.mon.b.smithi172.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-02-07T12:19:09.596 INFO:journalctl@ceph.mon.b.smithi172.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-02-07T12:19:09.596 INFO:journalctl@ceph.mon.b.smithi172.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-07T12:19:09.596 INFO:journalctl@ceph.mon.b.smithi172.stdout: 2024-02-07T12:19:09.596 INFO:journalctl@ceph.mon.b.smithi172.stdout: ** Compaction Stats [default] ** 2024-02-07T12:19:09.596 INFO:journalctl@ceph.mon.b.smithi172.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-07T12:19:09.596 INFO:journalctl@ceph.mon.b.smithi172.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-07T12:19:09.596 INFO:journalctl@ceph.mon.b.smithi172.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-02-07T12:19:09.596 INFO:journalctl@ceph.mon.b.smithi172.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-02-07T12:19:09.596 INFO:journalctl@ceph.mon.b.smithi172.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-02-07T12:19:09.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: 2024-02-07T12:19:09.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: ** Compaction Stats [default] ** 2024-02-07T12:19:09.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-07T12:19:09.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-07T12:19:09.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-02-07T12:19:09.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-02-07T12:19:09.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-02-07T12:19:09.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-02-07T12:19:09.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-02-07T12:19:09.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-02-07T12:19:09.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: AddFile(Keys): cumulative 0, interval 0 2024-02-07T12:19:09.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: Cumulative compaction: 0.00 GB write, 0.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:19:09.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: Interval compaction: 0.00 GB write, 0.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:19:09.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-02-07T12:19:09.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: 2024-02-07T12:19:09.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: ** File Read Latency Histogram By Level [default] ** 2024-02-07T12:19:09.598 INFO:journalctl@ceph.mon.b.smithi172.stdout: 2024-02-07T12:19:09.598 INFO:journalctl@ceph.mon.b.smithi172.stdout: ** Compaction Stats [default] ** 2024-02-07T12:19:09.598 INFO:journalctl@ceph.mon.b.smithi172.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-07T12:19:09.598 INFO:journalctl@ceph.mon.b.smithi172.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-07T12:19:09.598 INFO:journalctl@ceph.mon.b.smithi172.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-02-07T12:19:09.598 INFO:journalctl@ceph.mon.b.smithi172.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-02-07T12:19:09.598 INFO:journalctl@ceph.mon.b.smithi172.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-02-07T12:19:09.598 INFO:journalctl@ceph.mon.b.smithi172.stdout: 2024-02-07T12:19:09.598 INFO:journalctl@ceph.mon.b.smithi172.stdout: ** Compaction Stats [default] ** 2024-02-07T12:19:09.598 INFO:journalctl@ceph.mon.b.smithi172.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-07T12:19:09.598 INFO:journalctl@ceph.mon.b.smithi172.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-07T12:19:09.598 INFO:journalctl@ceph.mon.b.smithi172.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-02-07T12:19:09.598 INFO:journalctl@ceph.mon.b.smithi172.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-02-07T12:19:09.598 INFO:journalctl@ceph.mon.b.smithi172.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-02-07T12:19:09.598 INFO:journalctl@ceph.mon.b.smithi172.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-02-07T12:19:09.599 INFO:journalctl@ceph.mon.b.smithi172.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-02-07T12:19:09.599 INFO:journalctl@ceph.mon.b.smithi172.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-02-07T12:19:09.599 INFO:journalctl@ceph.mon.b.smithi172.stdout: AddFile(Keys): cumulative 0, interval 0 2024-02-07T12:19:09.599 INFO:journalctl@ceph.mon.b.smithi172.stdout: Cumulative compaction: 0.00 GB write, 0.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:19:09.599 INFO:journalctl@ceph.mon.b.smithi172.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:19:09.599 INFO:journalctl@ceph.mon.b.smithi172.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-02-07T12:19:09.599 INFO:journalctl@ceph.mon.b.smithi172.stdout: 2024-02-07T12:19:09.599 INFO:journalctl@ceph.mon.b.smithi172.stdout: ** File Read Latency Histogram By Level [default] ** 2024-02-07T12:19:09.599 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-02-07T12:19:09.599 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: using public_addr v2:172.21.15.172:0/0 -> [v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0] 2024-02-07T12:19:09.599 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: starting mon.b rank -1 at public addrs [v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0] at bind addrs [v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:19:09.599 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: mon.b@-1(???) e0 preinit fsid ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:19:09.599 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: mon.b@-1(synchronizing).mds e1 new map 2024-02-07T12:19:09.599 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: mon.b@-1(synchronizing).mds e1 print_map 2024-02-07T12:19:09.599 INFO:journalctl@ceph.mon.b.smithi172.stdout: e1 2024-02-07T12:19:09.600 INFO:journalctl@ceph.mon.b.smithi172.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-02-07T12:19:09.600 INFO:journalctl@ceph.mon.b.smithi172.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-02-07T12:19:09.600 INFO:journalctl@ceph.mon.b.smithi172.stdout: legacy client fscid: -1 2024-02-07T12:19:09.600 INFO:journalctl@ceph.mon.b.smithi172.stdout: 2024-02-07T12:19:09.600 INFO:journalctl@ceph.mon.b.smithi172.stdout: No filesystems configured 2024-02-07T12:19:09.600 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-02-07T12:19:09.600 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-02-07T12:19:09.600 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-02-07T12:19:09.600 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-02-07T12:19:09.600 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-02-07T12:19:09.600 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-02-07T12:19:09.600 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-02-07T12:19:09.600 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-02-07T12:19:09.600 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-02-07T12:19:09.600 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-02-07T12:19:09.601 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: Deploying daemon mon.c on smithi019 2024-02-07T12:19:09.601 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:09.601 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: from='client.? 172.21.15.172:0/3887620435' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:19:09.601 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:19:09.601 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: mon.a calling monitor election 2024-02-07T12:19:09.601 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:19:09.601 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:19:09.601 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:09.601 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:19:09.601 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: mon.c calling monitor election 2024-02-07T12:19:09.601 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:19:09.601 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:09.601 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:19:09.601 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:19:09.601 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-02-07T12:19:09.602 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: monmap e2: 2 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} 2024-02-07T12:19:09.602 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: fsmap 2024-02-07T12:19:09.602 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: osdmap e4: 0 total, 0 up, 0 in 2024-02-07T12:19:09.602 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: mgrmap e13: y(active, since 36s) 2024-02-07T12:19:09.602 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: overall HEALTH_OK 2024-02-07T12:19:09.602 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:09.602 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:19:09.602 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:09.602 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: Deploying daemon mon.b on smithi172 2024-02-07T12:19:09.602 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: from='client.? 172.21.15.172:0/129446763' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:19:09.602 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:09.602 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:19:09.602 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:09.602 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-02-07T12:19:09.602 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 ceph-mon[25457]: mon.b@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-02-07T12:19:09.603 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 bash[25424]: 7d62c7c4d075b789ea2a026b6848c89c9273a14590b88e7050e7e1d8a37ff299 2024-02-07T12:19:09.603 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:09 smithi172 systemd[1]: Started Ceph mon.b for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:19:10.604 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-02-07T12:19:10.605 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph mon dump -f json 2024-02-07T12:19:14.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:19:14.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:19:14.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[24574]: mon.a calling monitor election 2024-02-07T12:19:14.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[24574]: mon.c calling monitor election 2024-02-07T12:19:14.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:19:14.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:19:14.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[24574]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:14.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:19:14.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:19:14.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[24574]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:14.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:19:14.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:19:14.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[24574]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-02-07T12:19:14.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[24574]: monmap e3: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} 2024-02-07T12:19:14.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[24574]: fsmap 2024-02-07T12:19:14.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[24574]: osdmap e4: 0 total, 0 up, 0 in 2024-02-07T12:19:14.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[24574]: mgrmap e13: y(active, since 46s) 2024-02-07T12:19:14.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[24574]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-02-07T12:19:14.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[24574]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-02-07T12:19:14.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[24574]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-02-07T12:19:14.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[24574]: mon.b (rank 2) addr [v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0] is down (out of quorum) 2024-02-07T12:19:14.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:14.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:14.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:14.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:14.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:19:14.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:19:14.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[29267]: mon.a calling monitor election 2024-02-07T12:19:14.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[29267]: mon.c calling monitor election 2024-02-07T12:19:14.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:19:14.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:19:14.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[29267]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:14.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:19:14.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:19:14.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[29267]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:14.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:19:14.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:19:14.598 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[29267]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-02-07T12:19:14.598 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[29267]: monmap e3: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} 2024-02-07T12:19:14.598 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[29267]: fsmap 2024-02-07T12:19:14.598 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[29267]: osdmap e4: 0 total, 0 up, 0 in 2024-02-07T12:19:14.598 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[29267]: mgrmap e13: y(active, since 46s) 2024-02-07T12:19:14.598 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[29267]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-02-07T12:19:14.598 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[29267]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-02-07T12:19:14.598 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[29267]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-02-07T12:19:14.598 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[29267]: mon.b (rank 2) addr [v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0] is down (out of quorum) 2024-02-07T12:19:14.598 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:14.598 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:14.598 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:14.598 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:14 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:15.446 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-02-07T12:19:15.446 INFO:teuthology.orchestra.run.smithi172.stdout:{"epoch":3,"fsid":"ced2783c-c5b2-11ee-95b6-87774f69a715","modified":"2024-02-07T12:19:09.231435Z","created":"2024-02-07T12:17:35.503342Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:3300","nonce":0},{"type":"v1","addr":"172.21.15.19:6789","nonce":0}]},"addr":"172.21.15.19:6789/0","public_addr":"172.21.15.19:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:3301","nonce":0},{"type":"v1","addr":"172.21.15.19:6790","nonce":0}]},"addr":"172.21.15.19:6790/0","public_addr":"172.21.15.19:6790/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:3300","nonce":0},{"type":"v1","addr":"172.21.15.172:6789","nonce":0}]},"addr":"172.21.15.172:6789/0","public_addr":"172.21.15.172:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-02-07T12:19:15.448 INFO:teuthology.orchestra.run.smithi172.stderr:dumped monmap epoch 3 2024-02-07T12:19:15.550 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:15 smithi019 ceph-mon[24574]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:15.550 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:15 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:15.550 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:15 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:19:15.550 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:15 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:15.550 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:15 smithi019 ceph-mon[29267]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:15.551 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:15 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:15.551 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:15 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:19:15.551 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:15 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:15.973 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-02-07T12:19:15.973 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph config generate-minimal-conf 2024-02-07T12:19:16.568 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:19:16.568 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:19:16.568 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: mon.a calling monitor election 2024-02-07T12:19:16.568 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: mon.c calling monitor election 2024-02-07T12:19:16.568 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:19:16.568 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:19:16.568 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:16.568 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:19:16.568 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:19:16.569 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:16.569 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:19:16.569 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:19:16.569 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-02-07T12:19:16.569 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: monmap e3: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} 2024-02-07T12:19:16.569 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: fsmap 2024-02-07T12:19:16.569 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: osdmap e4: 0 total, 0 up, 0 in 2024-02-07T12:19:16.569 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: mgrmap e13: y(active, since 46s) 2024-02-07T12:19:16.569 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-02-07T12:19:16.569 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-02-07T12:19:16.569 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-02-07T12:19:16.569 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: mon.b (rank 2) addr [v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0] is down (out of quorum) 2024-02-07T12:19:16.569 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:16.569 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:16.569 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:16.570 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:16.570 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:16.570 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:16.570 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:19:16.570 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:16.570 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: mon.b calling monitor election 2024-02-07T12:19:16.570 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: mon.b calling monitor election 2024-02-07T12:19:16.570 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: mon.c calling monitor election 2024-02-07T12:19:16.570 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: mon.a calling monitor election 2024-02-07T12:19:16.570 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-02-07T12:19:16.570 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: monmap e3: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} 2024-02-07T12:19:16.570 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: fsmap 2024-02-07T12:19:16.570 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: osdmap e4: 0 total, 0 up, 0 in 2024-02-07T12:19:16.570 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: mgrmap e13: y(active, since 48s) 2024-02-07T12:19:16.570 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-02-07T12:19:16.570 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: Cluster is now healthy 2024-02-07T12:19:16.571 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:16 smithi172 ceph-mon[25457]: overall HEALTH_OK 2024-02-07T12:19:16.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:16 smithi019 ceph-mon[24574]: mon.b calling monitor election 2024-02-07T12:19:16.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:16 smithi019 ceph-mon[24574]: mon.b calling monitor election 2024-02-07T12:19:16.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:16 smithi019 ceph-mon[24574]: mon.c calling monitor election 2024-02-07T12:19:16.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:16 smithi019 ceph-mon[24574]: mon.a calling monitor election 2024-02-07T12:19:16.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:16 smithi019 ceph-mon[24574]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-02-07T12:19:16.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:16 smithi019 ceph-mon[24574]: monmap e3: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} 2024-02-07T12:19:16.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:16 smithi019 ceph-mon[24574]: fsmap 2024-02-07T12:19:16.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:16 smithi019 ceph-mon[24574]: osdmap e4: 0 total, 0 up, 0 in 2024-02-07T12:19:16.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:16 smithi019 ceph-mon[24574]: mgrmap e13: y(active, since 48s) 2024-02-07T12:19:16.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:16 smithi019 ceph-mon[24574]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-02-07T12:19:16.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:16 smithi019 ceph-mon[24574]: Cluster is now healthy 2024-02-07T12:19:16.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:16 smithi019 ceph-mon[24574]: overall HEALTH_OK 2024-02-07T12:19:16.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:16 smithi019 ceph-mon[29267]: mon.b calling monitor election 2024-02-07T12:19:16.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:16 smithi019 ceph-mon[29267]: mon.b calling monitor election 2024-02-07T12:19:16.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:16 smithi019 ceph-mon[29267]: mon.c calling monitor election 2024-02-07T12:19:16.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:16 smithi019 ceph-mon[29267]: mon.a calling monitor election 2024-02-07T12:19:16.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:16 smithi019 ceph-mon[29267]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-02-07T12:19:16.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:16 smithi019 ceph-mon[29267]: monmap e3: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} 2024-02-07T12:19:16.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:16 smithi019 ceph-mon[29267]: fsmap 2024-02-07T12:19:16.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:16 smithi019 ceph-mon[29267]: osdmap e4: 0 total, 0 up, 0 in 2024-02-07T12:19:16.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:16 smithi019 ceph-mon[29267]: mgrmap e13: y(active, since 48s) 2024-02-07T12:19:16.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:16 smithi019 ceph-mon[29267]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-02-07T12:19:16.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:16 smithi019 ceph-mon[29267]: Cluster is now healthy 2024-02-07T12:19:16.847 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:16 smithi019 ceph-mon[29267]: overall HEALTH_OK 2024-02-07T12:19:17.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:17 smithi019 ceph-mon[24574]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:17.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:17 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:19:17.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:17 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:17.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:17 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:19:17.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:17 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:19:17.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:17 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:17.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:17 smithi019 ceph-mon[29267]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:17.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:17 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:19:17.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:17 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:17.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:17 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:19:17.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:17 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:19:17.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:17 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:17.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:17 smithi172 ceph-mon[25457]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:17.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:17 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:19:17.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:17 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:17.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:17 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:19:17.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:17 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:19:17.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:17 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:18.518 INFO:teuthology.orchestra.run.smithi019.stdout:# minimal ceph.conf for ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:19:18.518 INFO:teuthology.orchestra.run.smithi019.stdout:[global] 2024-02-07T12:19:18.518 INFO:teuthology.orchestra.run.smithi019.stdout: fsid = ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:19:18.518 INFO:teuthology.orchestra.run.smithi019.stdout: mon_host = [v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0] [v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0] [v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0] 2024-02-07T12:19:18.787 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:18 smithi019 ceph-mon[29267]: Reconfiguring mon.c (monmap changed)... 2024-02-07T12:19:18.787 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:18 smithi019 ceph-mon[29267]: Reconfiguring daemon mon.c on smithi019 2024-02-07T12:19:18.787 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:18 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1413102812' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:18.788 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:18 smithi019 ceph-mon[24574]: Reconfiguring mon.c (monmap changed)... 2024-02-07T12:19:18.788 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:18 smithi019 ceph-mon[24574]: Reconfiguring daemon mon.c on smithi019 2024-02-07T12:19:18.788 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:18 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1413102812' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:19.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:18 smithi172 ceph-mon[25457]: Reconfiguring mon.c (monmap changed)... 2024-02-07T12:19:19.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:18 smithi172 ceph-mon[25457]: Reconfiguring daemon mon.c on smithi019 2024-02-07T12:19:19.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:18 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1413102812' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:19.160 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-02-07T12:19:19.160 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2024-02-07T12:19:19.160 DEBUG:teuthology.orchestra.run.smithi019:> sudo dd of=/etc/ceph/ceph.conf 2024-02-07T12:19:19.189 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2024-02-07T12:19:19.189 DEBUG:teuthology.orchestra.run.smithi019:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:19:19.255 DEBUG:teuthology.orchestra.run.smithi172:> set -ex 2024-02-07T12:19:19.255 DEBUG:teuthology.orchestra.run.smithi172:> sudo dd of=/etc/ceph/ceph.conf 2024-02-07T12:19:19.291 DEBUG:teuthology.orchestra.run.smithi172:> set -ex 2024-02-07T12:19:19.291 DEBUG:teuthology.orchestra.run.smithi172:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:19:19.359 INFO:tasks.cephadm:Adding mgr.y on smithi019 2024-02-07T12:19:19.359 INFO:tasks.cephadm:Adding mgr.x on smithi172 2024-02-07T12:19:19.360 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph orch apply mgr '2;smithi019=y;smithi172=x' 2024-02-07T12:19:20.000 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:19 smithi019 ceph-mon[29267]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:20.000 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:19 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:20.000 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:19 smithi019 ceph-mon[24574]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:20.000 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:19 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:20.089 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:19 smithi172 ceph-mon[25457]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:20.089 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:19 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:21.336 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:21 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:21.336 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:21 smithi172 ceph-mon[25457]: Reconfiguring mon.b (monmap changed)... 2024-02-07T12:19:21.336 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:21 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:19:21.337 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:21 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:19:21.337 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:21 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:21.337 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:21 smithi172 ceph-mon[25457]: Reconfiguring daemon mon.b on smithi172 2024-02-07T12:19:21.337 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:21 smithi172 ceph-mon[25457]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:21.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:21 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:21.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:21 smithi019 ceph-mon[24574]: Reconfiguring mon.b (monmap changed)... 2024-02-07T12:19:21.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:21 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:19:21.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:21 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:19:21.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:21 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:21.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:21 smithi019 ceph-mon[24574]: Reconfiguring daemon mon.b on smithi172 2024-02-07T12:19:21.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:21 smithi019 ceph-mon[24574]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:21.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:21 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:21.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:21 smithi019 ceph-mon[29267]: Reconfiguring mon.b (monmap changed)... 2024-02-07T12:19:21.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:21 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:19:21.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:21 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:19:21.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:21 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:21.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:21 smithi019 ceph-mon[29267]: Reconfiguring daemon mon.b on smithi172 2024-02-07T12:19:21.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:21 smithi019 ceph-mon[29267]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:21.626 INFO:teuthology.orchestra.run.smithi172.stdout:Scheduled mgr update... 2024-02-07T12:19:22.274 DEBUG:teuthology.orchestra.run.smithi172:mgr.x> sudo journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mgr.x.service 2024-02-07T12:19:22.277 INFO:tasks.cephadm:Deploying OSDs... 2024-02-07T12:19:22.277 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2024-02-07T12:19:22.277 DEBUG:teuthology.orchestra.run.smithi019:> dd if=/scratch_devs of=/dev/stdout 2024-02-07T12:19:22.297 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-02-07T12:19:22.297 DEBUG:teuthology.orchestra.run.smithi019:> stat /dev/vg_nvme/lv_1 2024-02-07T12:19:22.355 INFO:teuthology.orchestra.run.smithi019.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-02-07T12:19:22.356 INFO:teuthology.orchestra.run.smithi019.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-02-07T12:19:22.356 INFO:teuthology.orchestra.run.smithi019.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-02-07T12:19:22.357 INFO:teuthology.orchestra.run.smithi019.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-02-07T12:19:22.357 INFO:teuthology.orchestra.run.smithi019.stdout:Context: system_u:object_r:device_t:s0 2024-02-07T12:19:22.357 INFO:teuthology.orchestra.run.smithi019.stdout:Access: 2024-02-07 12:18:58.683852301 +0000 2024-02-07T12:19:22.357 INFO:teuthology.orchestra.run.smithi019.stdout:Modify: 2024-02-07 12:16:43.219083078 +0000 2024-02-07T12:19:22.357 INFO:teuthology.orchestra.run.smithi019.stdout:Change: 2024-02-07 12:16:43.219083078 +0000 2024-02-07T12:19:22.357 INFO:teuthology.orchestra.run.smithi019.stdout: Birth: 2024-02-07 12:16:43.219083078 +0000 2024-02-07T12:19:22.357 DEBUG:teuthology.orchestra.run.smithi019:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-02-07T12:19:22.428 INFO:teuthology.orchestra.run.smithi019.stderr:1+0 records in 2024-02-07T12:19:22.428 INFO:teuthology.orchestra.run.smithi019.stderr:1+0 records out 2024-02-07T12:19:22.428 INFO:teuthology.orchestra.run.smithi019.stderr:512 bytes copied, 0.000256781 s, 2.0 MB/s 2024-02-07T12:19:22.429 DEBUG:teuthology.orchestra.run.smithi019:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-02-07T12:19:22.489 DEBUG:teuthology.orchestra.run.smithi019:> stat /dev/vg_nvme/lv_2 2024-02-07T12:19:22.547 INFO:teuthology.orchestra.run.smithi019.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-02-07T12:19:22.547 INFO:teuthology.orchestra.run.smithi019.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-02-07T12:19:22.548 INFO:teuthology.orchestra.run.smithi019.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-02-07T12:19:22.548 INFO:teuthology.orchestra.run.smithi019.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-02-07T12:19:22.548 INFO:teuthology.orchestra.run.smithi019.stdout:Context: system_u:object_r:device_t:s0 2024-02-07T12:19:22.548 INFO:teuthology.orchestra.run.smithi019.stdout:Access: 2024-02-07 12:18:58.684852291 +0000 2024-02-07T12:19:22.548 INFO:teuthology.orchestra.run.smithi019.stdout:Modify: 2024-02-07 12:16:43.561079656 +0000 2024-02-07T12:19:22.548 INFO:teuthology.orchestra.run.smithi019.stdout:Change: 2024-02-07 12:16:43.561079656 +0000 2024-02-07T12:19:22.548 INFO:teuthology.orchestra.run.smithi019.stdout: Birth: 2024-02-07 12:16:43.561079656 +0000 2024-02-07T12:19:22.548 DEBUG:teuthology.orchestra.run.smithi019:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-02-07T12:19:22.617 INFO:teuthology.orchestra.run.smithi019.stderr:1+0 records in 2024-02-07T12:19:22.617 INFO:teuthology.orchestra.run.smithi019.stderr:1+0 records out 2024-02-07T12:19:22.618 INFO:teuthology.orchestra.run.smithi019.stderr:512 bytes copied, 0.0002001 s, 2.6 MB/s 2024-02-07T12:19:22.619 DEBUG:teuthology.orchestra.run.smithi019:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-02-07T12:19:22.678 DEBUG:teuthology.orchestra.run.smithi019:> stat /dev/vg_nvme/lv_3 2024-02-07T12:19:22.736 INFO:teuthology.orchestra.run.smithi019.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-02-07T12:19:22.736 INFO:teuthology.orchestra.run.smithi019.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-02-07T12:19:22.736 INFO:teuthology.orchestra.run.smithi019.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-02-07T12:19:22.737 INFO:teuthology.orchestra.run.smithi019.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-02-07T12:19:22.737 INFO:teuthology.orchestra.run.smithi019.stdout:Context: system_u:object_r:device_t:s0 2024-02-07T12:19:22.737 INFO:teuthology.orchestra.run.smithi019.stdout:Access: 2024-02-07 12:18:58.684852291 +0000 2024-02-07T12:19:22.737 INFO:teuthology.orchestra.run.smithi019.stdout:Modify: 2024-02-07 12:16:43.910076163 +0000 2024-02-07T12:19:22.737 INFO:teuthology.orchestra.run.smithi019.stdout:Change: 2024-02-07 12:16:43.910076163 +0000 2024-02-07T12:19:22.737 INFO:teuthology.orchestra.run.smithi019.stdout: Birth: 2024-02-07 12:16:43.910076163 +0000 2024-02-07T12:19:22.737 DEBUG:teuthology.orchestra.run.smithi019:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-02-07T12:19:22.803 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:22 smithi019 ceph-mon[29267]: from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi019=y;smithi172=x", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:19:22.803 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:22 smithi019 ceph-mon[29267]: Saving service mgr spec with placement smithi019=y;smithi172=x;count:2 2024-02-07T12:19:22.803 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:22 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:22.803 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:22 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:22.803 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:22 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:22.803 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:22 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:22.803 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:22 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:22.803 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:22 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:22.803 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:22 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:19:22.803 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:22 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-02-07T12:19:22.803 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:22 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:19:22.803 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:22 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:22.804 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:22 smithi019 ceph-mon[29267]: Deploying daemon mgr.x on smithi172 2024-02-07T12:19:22.804 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:22 smithi019 ceph-mon[24574]: from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi019=y;smithi172=x", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:19:22.804 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:22 smithi019 ceph-mon[24574]: Saving service mgr spec with placement smithi019=y;smithi172=x;count:2 2024-02-07T12:19:22.804 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:22 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:22.804 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:22 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:22.804 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:22 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:22.804 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:22 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:22.804 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:22 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:22.804 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:22 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:22.804 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:22 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:19:22.804 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:22 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-02-07T12:19:22.804 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:22 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:19:22.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:22 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:22.805 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:22 smithi019 ceph-mon[24574]: Deploying daemon mgr.x on smithi172 2024-02-07T12:19:22.805 INFO:teuthology.orchestra.run.smithi019.stderr:1+0 records in 2024-02-07T12:19:22.805 INFO:teuthology.orchestra.run.smithi019.stderr:1+0 records out 2024-02-07T12:19:22.805 INFO:teuthology.orchestra.run.smithi019.stderr:512 bytes copied, 0.00019085 s, 2.7 MB/s 2024-02-07T12:19:22.806 DEBUG:teuthology.orchestra.run.smithi019:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-02-07T12:19:22.865 DEBUG:teuthology.orchestra.run.smithi019:> stat /dev/vg_nvme/lv_4 2024-02-07T12:19:22.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:22 smithi172 ceph-mon[25457]: from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi019=y;smithi172=x", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:19:22.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:22 smithi172 ceph-mon[25457]: Saving service mgr spec with placement smithi019=y;smithi172=x;count:2 2024-02-07T12:19:22.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:22 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:22.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:22 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:22.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:22 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:22.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:22 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:22.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:22 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:22.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:22 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:22.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:22 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:19:22.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:22 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-02-07T12:19:22.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:22 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:19:22.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:22 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:22.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:22 smithi172 ceph-mon[25457]: Deploying daemon mgr.x on smithi172 2024-02-07T12:19:22.922 INFO:teuthology.orchestra.run.smithi019.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-02-07T12:19:22.922 INFO:teuthology.orchestra.run.smithi019.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-02-07T12:19:22.922 INFO:teuthology.orchestra.run.smithi019.stdout:Device: 5h/5d Inode: 725 Links: 1 2024-02-07T12:19:22.922 INFO:teuthology.orchestra.run.smithi019.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-02-07T12:19:22.922 INFO:teuthology.orchestra.run.smithi019.stdout:Context: system_u:object_r:device_t:s0 2024-02-07T12:19:22.922 INFO:teuthology.orchestra.run.smithi019.stdout:Access: 2024-02-07 12:18:58.685852281 +0000 2024-02-07T12:19:22.923 INFO:teuthology.orchestra.run.smithi019.stdout:Modify: 2024-02-07 12:16:44.249072771 +0000 2024-02-07T12:19:22.923 INFO:teuthology.orchestra.run.smithi019.stdout:Change: 2024-02-07 12:16:44.249072771 +0000 2024-02-07T12:19:22.923 INFO:teuthology.orchestra.run.smithi019.stdout: Birth: 2024-02-07 12:16:44.249072771 +0000 2024-02-07T12:19:22.923 DEBUG:teuthology.orchestra.run.smithi019:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-02-07T12:19:22.990 INFO:teuthology.orchestra.run.smithi019.stderr:1+0 records in 2024-02-07T12:19:22.990 INFO:teuthology.orchestra.run.smithi019.stderr:1+0 records out 2024-02-07T12:19:22.990 INFO:teuthology.orchestra.run.smithi019.stderr:512 bytes copied, 0.000205658 s, 2.5 MB/s 2024-02-07T12:19:22.991 DEBUG:teuthology.orchestra.run.smithi019:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-02-07T12:19:23.048 DEBUG:teuthology.orchestra.run.smithi172:> set -ex 2024-02-07T12:19:23.049 DEBUG:teuthology.orchestra.run.smithi172:> dd if=/scratch_devs of=/dev/stdout 2024-02-07T12:19:23.065 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-02-07T12:19:23.066 DEBUG:teuthology.orchestra.run.smithi172:> stat /dev/vg_nvme/lv_1 2024-02-07T12:19:23.121 INFO:teuthology.orchestra.run.smithi172.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-02-07T12:19:23.121 INFO:teuthology.orchestra.run.smithi172.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-02-07T12:19:23.122 INFO:teuthology.orchestra.run.smithi172.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-02-07T12:19:23.122 INFO:teuthology.orchestra.run.smithi172.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-02-07T12:19:23.122 INFO:teuthology.orchestra.run.smithi172.stdout:Context: system_u:object_r:device_t:s0 2024-02-07T12:19:23.122 INFO:teuthology.orchestra.run.smithi172.stdout:Access: 2024-02-07 12:19:07.932791949 +0000 2024-02-07T12:19:23.122 INFO:teuthology.orchestra.run.smithi172.stdout:Modify: 2024-02-07 12:16:41.656773721 +0000 2024-02-07T12:19:23.122 INFO:teuthology.orchestra.run.smithi172.stdout:Change: 2024-02-07 12:16:41.656773721 +0000 2024-02-07T12:19:23.122 INFO:teuthology.orchestra.run.smithi172.stdout: Birth: 2024-02-07 12:16:41.656773721 +0000 2024-02-07T12:19:23.122 DEBUG:teuthology.orchestra.run.smithi172:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-02-07T12:19:23.188 INFO:teuthology.orchestra.run.smithi172.stderr:1+0 records in 2024-02-07T12:19:23.188 INFO:teuthology.orchestra.run.smithi172.stderr:1+0 records out 2024-02-07T12:19:23.188 INFO:teuthology.orchestra.run.smithi172.stderr:512 bytes copied, 0.000191863 s, 2.7 MB/s 2024-02-07T12:19:23.190 DEBUG:teuthology.orchestra.run.smithi172:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-02-07T12:19:23.246 DEBUG:teuthology.orchestra.run.smithi172:> stat /dev/vg_nvme/lv_2 2024-02-07T12:19:23.301 INFO:teuthology.orchestra.run.smithi172.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-02-07T12:19:23.301 INFO:teuthology.orchestra.run.smithi172.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-02-07T12:19:23.301 INFO:teuthology.orchestra.run.smithi172.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-02-07T12:19:23.301 INFO:teuthology.orchestra.run.smithi172.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-02-07T12:19:23.301 INFO:teuthology.orchestra.run.smithi172.stdout:Context: system_u:object_r:device_t:s0 2024-02-07T12:19:23.301 INFO:teuthology.orchestra.run.smithi172.stdout:Access: 2024-02-07 12:19:07.932791949 +0000 2024-02-07T12:19:23.301 INFO:teuthology.orchestra.run.smithi172.stdout:Modify: 2024-02-07 12:16:42.002765230 +0000 2024-02-07T12:19:23.301 INFO:teuthology.orchestra.run.smithi172.stdout:Change: 2024-02-07 12:16:42.002765230 +0000 2024-02-07T12:19:23.301 INFO:teuthology.orchestra.run.smithi172.stdout: Birth: 2024-02-07 12:16:42.002765230 +0000 2024-02-07T12:19:23.302 DEBUG:teuthology.orchestra.run.smithi172:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-02-07T12:19:23.367 INFO:teuthology.orchestra.run.smithi172.stderr:1+0 records in 2024-02-07T12:19:23.367 INFO:teuthology.orchestra.run.smithi172.stderr:1+0 records out 2024-02-07T12:19:23.367 INFO:teuthology.orchestra.run.smithi172.stderr:512 bytes copied, 0.000184656 s, 2.8 MB/s 2024-02-07T12:19:23.368 DEBUG:teuthology.orchestra.run.smithi172:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-02-07T12:19:23.424 DEBUG:teuthology.orchestra.run.smithi172:> stat /dev/vg_nvme/lv_3 2024-02-07T12:19:23.480 INFO:teuthology.orchestra.run.smithi172.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-02-07T12:19:23.480 INFO:teuthology.orchestra.run.smithi172.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-02-07T12:19:23.480 INFO:teuthology.orchestra.run.smithi172.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-02-07T12:19:23.481 INFO:teuthology.orchestra.run.smithi172.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-02-07T12:19:23.481 INFO:teuthology.orchestra.run.smithi172.stdout:Context: system_u:object_r:device_t:s0 2024-02-07T12:19:23.481 INFO:teuthology.orchestra.run.smithi172.stdout:Access: 2024-02-07 12:19:07.933791925 +0000 2024-02-07T12:19:23.481 INFO:teuthology.orchestra.run.smithi172.stdout:Modify: 2024-02-07 12:16:42.332757131 +0000 2024-02-07T12:19:23.481 INFO:teuthology.orchestra.run.smithi172.stdout:Change: 2024-02-07 12:16:42.332757131 +0000 2024-02-07T12:19:23.481 INFO:teuthology.orchestra.run.smithi172.stdout: Birth: 2024-02-07 12:16:42.332757131 +0000 2024-02-07T12:19:23.481 DEBUG:teuthology.orchestra.run.smithi172:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-02-07T12:19:23.568 INFO:teuthology.orchestra.run.smithi172.stderr:1+0 records in 2024-02-07T12:19:23.568 INFO:teuthology.orchestra.run.smithi172.stderr:1+0 records out 2024-02-07T12:19:23.568 INFO:teuthology.orchestra.run.smithi172.stderr:512 bytes copied, 0.000198292 s, 2.6 MB/s 2024-02-07T12:19:23.570 DEBUG:teuthology.orchestra.run.smithi172:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-02-07T12:19:23.585 DEBUG:teuthology.orchestra.run.smithi172:> stat /dev/vg_nvme/lv_4 2024-02-07T12:19:23.641 INFO:teuthology.orchestra.run.smithi172.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-02-07T12:19:23.641 INFO:teuthology.orchestra.run.smithi172.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-02-07T12:19:23.641 INFO:teuthology.orchestra.run.smithi172.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-02-07T12:19:23.641 INFO:teuthology.orchestra.run.smithi172.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-02-07T12:19:23.641 INFO:teuthology.orchestra.run.smithi172.stdout:Context: system_u:object_r:device_t:s0 2024-02-07T12:19:23.641 INFO:teuthology.orchestra.run.smithi172.stdout:Access: 2024-02-07 12:19:23.526421660 +0000 2024-02-07T12:19:23.641 INFO:teuthology.orchestra.run.smithi172.stdout:Modify: 2024-02-07 12:16:42.659749106 +0000 2024-02-07T12:19:23.641 INFO:teuthology.orchestra.run.smithi172.stdout:Change: 2024-02-07 12:16:42.659749106 +0000 2024-02-07T12:19:23.641 INFO:teuthology.orchestra.run.smithi172.stdout: Birth: 2024-02-07 12:16:42.659749106 +0000 2024-02-07T12:19:23.642 DEBUG:teuthology.orchestra.run.smithi172:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-02-07T12:19:23.743 INFO:teuthology.orchestra.run.smithi172.stderr:1+0 records in 2024-02-07T12:19:23.743 INFO:teuthology.orchestra.run.smithi172.stderr:1+0 records out 2024-02-07T12:19:23.743 INFO:teuthology.orchestra.run.smithi172.stderr:512 bytes copied, 0.00018234 s, 2.8 MB/s 2024-02-07T12:19:23.744 DEBUG:teuthology.orchestra.run.smithi172:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-02-07T12:19:23.759 INFO:tasks.cephadm:Deploying osd.0 on smithi019 with /dev/vg_nvme/lv_4... 2024-02-07T12:19:23.759 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-02-07T12:19:23.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:23 smithi172 ceph-mon[25457]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:23.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:23 smithi172 systemd[1]: Starting Ceph mgr.x for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:19:24.079 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:23 smithi019 ceph-mon[24574]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:24.079 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:23 smithi019 ceph-mon[29267]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:24.245 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:23 smithi172 podman[26626]: 2024-02-07 12:19:23.965897154 +0000 UTC m=+0.103255804 container create 3ce4fcd95143d8b5bc16a2f80bf7ef24e04f500cb7768ceedec624afd4898c24 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x, release=754, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, name=centos-stream, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , ceph=True, build-date=2022-05-03T08:36:31.336870, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8) 2024-02-07T12:19:24.245 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:23 smithi172 podman[26626]: 2024-02-07 12:19:23.881102427 +0000 UTC m=+0.018461079 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-02-07T12:19:24.245 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:24 smithi172 podman[26626]: 2024-02-07 12:19:24.144189773 +0000 UTC m=+0.281548422 container init 3ce4fcd95143d8b5bc16a2f80bf7ef24e04f500cb7768ceedec624afd4898c24 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x, version=8, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, vendor=Red Hat, Inc., name=centos-stream) 2024-02-07T12:19:24.245 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:24 smithi172 podman[26626]: 2024-02-07 12:19:24.146982192 +0000 UTC m=+0.284340849 container start 3ce4fcd95143d8b5bc16a2f80bf7ef24e04f500cb7768ceedec624afd4898c24 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, vcs-type=git, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, ceph=True, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., version=8, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-02-07T12:19:24.245 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:24 smithi172 ceph-mgr[26644]: -- 172.21.15.172:0/3864654809 <== mon.2 v2:172.21.15.172:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55558b318340 con 0x55558b2f0c00 2024-02-07T12:19:24.246 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:24 smithi172 bash[26626]: 3ce4fcd95143d8b5bc16a2f80bf7ef24e04f500cb7768ceedec624afd4898c24 2024-02-07T12:19:24.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:24 smithi172 systemd[1]: Started Ceph mgr.x for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:19:24.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:24 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:19:24.663+0000 7f313d684000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-02-07T12:19:24.969 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:24 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:19:24.758+0000 7f313d684000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-02-07T12:19:25.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:25 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:25.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:25 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:25.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:25 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:25.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:25 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:25.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:25 smithi019 ceph-mon[29267]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:25.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:25 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:25.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:25 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:25.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:25 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:25.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:25 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:25.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:25 smithi019 ceph-mon[24574]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:25.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:25 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:25.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:25 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:25.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:25 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:25.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:25 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:25.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:25 smithi172 ceph-mon[25457]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:25.620 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:19:25.374+0000 7f313d684000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-02-07T12:19:25.620 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:19:25.470+0000 7f313d684000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-02-07T12:19:25.620 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:19:25.553+0000 7f313d684000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-02-07T12:19:25.938 INFO:teuthology.orchestra.run.smithi019.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-02-07T12:19:25.938 INFO:teuthology.orchestra.run.smithi019.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-02-07T12:19:25.938 INFO:teuthology.orchestra.run.smithi019.stderr: stderr: 10+0 records in 2024-02-07T12:19:25.938 INFO:teuthology.orchestra.run.smithi019.stderr:10+0 records out 2024-02-07T12:19:25.938 INFO:teuthology.orchestra.run.smithi019.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0114481 s, 916 MB/s 2024-02-07T12:19:25.938 INFO:teuthology.orchestra.run.smithi019.stderr:--> Zapping successful for: 2024-02-07T12:19:25.977 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:19:25.702+0000 7f313d684000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-02-07T12:19:25.977 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:19:25.783+0000 7f313d684000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-02-07T12:19:25.978 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:19:25.892+0000 7f313d684000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-02-07T12:19:25.978 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:19:25.975+0000 7f313d684000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-02-07T12:19:26.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:26 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:19:26.139+0000 7f313d684000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-02-07T12:19:26.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:26 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:19:26.223+0000 7f313d684000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-02-07T12:19:26.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:26 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:26.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:26 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:26.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:26 smithi019 ceph-mon[24574]: Reconfiguring mgr.y (unknown last config time)... 2024-02-07T12:19:26.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:26 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:19:26.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:26 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:19:26.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:26 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:26.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:26 smithi019 ceph-mon[24574]: Reconfiguring daemon mgr.y on smithi019 2024-02-07T12:19:26.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:26 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:26.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:26 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:26.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:26 smithi019 ceph-mon[29267]: Reconfiguring mgr.y (unknown last config time)... 2024-02-07T12:19:26.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:26 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:19:26.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:26 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:19:26.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:26 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:26.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:26 smithi019 ceph-mon[29267]: Reconfiguring daemon mgr.y on smithi019 2024-02-07T12:19:26.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:26 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:26.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:26 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:26.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:26 smithi172 ceph-mon[25457]: Reconfiguring mgr.y (unknown last config time)... 2024-02-07T12:19:26.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:26 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:19:26.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:26 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:19:26.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:26 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:26.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:26 smithi172 ceph-mon[25457]: Reconfiguring daemon mgr.y on smithi019 2024-02-07T12:19:27.236 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph orch daemon add osd smithi019:vg_nvme/lv_4 2024-02-07T12:19:27.546 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:27 smithi019 ceph-mon[24574]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:27.546 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:27 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:27.546 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:27 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:27.546 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:27 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:27.546 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:27 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:27.547 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:27 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:27.547 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:27 smithi019 ceph-mon[29267]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:27.547 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:27 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:27.547 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:27 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:27.547 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:27 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:27.547 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:27 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:27.547 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:27 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:27.589 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:19:27.266+0000 7f313d684000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-02-07T12:19:27.590 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:19:27.395+0000 7f313d684000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-02-07T12:19:27.590 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:19:27.587+0000 7f313d684000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-02-07T12:19:27.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:27 smithi172 ceph-mon[25457]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:27.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:27 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:27.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:27 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:27.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:27 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:27.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:27 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:27.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:27 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:27.869 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:19:27.674+0000 7f313d684000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-02-07T12:19:27.869 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:19:27.809+0000 7f313d684000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:19:28.433 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:28 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:19:28.128+0000 7f313d684000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-02-07T12:19:28.433 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:28 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:19:28.347+0000 7f313d684000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:19:28.434 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:28 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:19:28.431+0000 7f313d684000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-02-07T12:19:28.732 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:28 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:19:28.601+0000 7f313d684000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-02-07T12:19:28.971 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:28 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:19:28.971 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:28 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:19:28.972 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:28 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:28.972 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:28 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:19:28.972 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:28 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:19:28.972 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:28 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:29.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:28 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:19:29.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:28 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:19:29.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:28 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:29.820 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:29 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:19:29.522+0000 7f313d684000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-02-07T12:19:29.820 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:29 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:19:29.818+0000 7f313d684000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-02-07T12:19:29.821 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:29 smithi172 ceph-mon[25457]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:29.821 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:29 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:19:29.821 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:29 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:19:29.821 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:29 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:29.924 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:29 smithi019 ceph-mon[24574]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:29.924 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:29 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:19:29.925 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:29 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:19:29.925 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:29 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:29.925 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:29 smithi019 ceph-mon[29267]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:29.925 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:29 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:19:29.925 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:29 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:19:29.925 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:29 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:30.119 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:29 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:19:29.913+0000 7f313d684000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-02-07T12:19:30.119 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:19:30 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:19:30.065+0000 7f313d684000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-02-07T12:19:30.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:30 smithi019 ceph-mon[29267]: from='client.24115 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi019:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:19:30.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:30 smithi019 ceph-mon[29267]: Standby manager daemon x started 2024-02-07T12:19:30.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:30 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/2471916748' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-02-07T12:19:30.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:30 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/2471916748' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:19:30.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:30 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/2471916748' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-02-07T12:19:30.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:30 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/2471916748' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:19:30.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:30 smithi019 ceph-mon[24574]: from='client.24115 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi019:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:19:30.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:30 smithi019 ceph-mon[24574]: Standby manager daemon x started 2024-02-07T12:19:30.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:30 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.172:0/2471916748' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-02-07T12:19:30.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:30 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.172:0/2471916748' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:19:30.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:30 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.172:0/2471916748' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-02-07T12:19:30.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:30 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.172:0/2471916748' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:19:31.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:30 smithi172 ceph-mon[25457]: from='client.24115 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi019:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:19:31.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:30 smithi172 ceph-mon[25457]: Standby manager daemon x started 2024-02-07T12:19:31.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:30 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.172:0/2471916748' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-02-07T12:19:31.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:30 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.172:0/2471916748' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:19:31.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:30 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.172:0/2471916748' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-02-07T12:19:31.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:30 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.172:0/2471916748' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:19:32.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:31 smithi019 ceph-mon[24574]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:32.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:31 smithi019 ceph-mon[24574]: mgrmap e14: y(active, since 62s), standbys: x 2024-02-07T12:19:32.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:31 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-02-07T12:19:32.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:31 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2274185445' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3ddfcd81-94a5-46f7-812e-c87f08345aef"}]: dispatch 2024-02-07T12:19:32.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:31 smithi019 ceph-mon[24574]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3ddfcd81-94a5-46f7-812e-c87f08345aef"}]: dispatch 2024-02-07T12:19:32.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:31 smithi019 ceph-mon[24574]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3ddfcd81-94a5-46f7-812e-c87f08345aef"}]': finished 2024-02-07T12:19:32.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:31 smithi019 ceph-mon[24574]: osdmap e5: 1 total, 0 up, 1 in 2024-02-07T12:19:32.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:31 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:19:32.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:31 smithi019 ceph-mon[29267]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:32.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:31 smithi019 ceph-mon[29267]: mgrmap e14: y(active, since 62s), standbys: x 2024-02-07T12:19:32.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:31 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-02-07T12:19:32.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:31 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2274185445' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3ddfcd81-94a5-46f7-812e-c87f08345aef"}]: dispatch 2024-02-07T12:19:32.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:31 smithi019 ceph-mon[29267]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3ddfcd81-94a5-46f7-812e-c87f08345aef"}]: dispatch 2024-02-07T12:19:32.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:31 smithi019 ceph-mon[29267]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3ddfcd81-94a5-46f7-812e-c87f08345aef"}]': finished 2024-02-07T12:19:32.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:31 smithi019 ceph-mon[29267]: osdmap e5: 1 total, 0 up, 1 in 2024-02-07T12:19:32.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:31 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:19:32.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:31 smithi172 ceph-mon[25457]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:32.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:31 smithi172 ceph-mon[25457]: mgrmap e14: y(active, since 62s), standbys: x 2024-02-07T12:19:32.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:31 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-02-07T12:19:32.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:31 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2274185445' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3ddfcd81-94a5-46f7-812e-c87f08345aef"}]: dispatch 2024-02-07T12:19:32.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:31 smithi172 ceph-mon[25457]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3ddfcd81-94a5-46f7-812e-c87f08345aef"}]: dispatch 2024-02-07T12:19:32.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:31 smithi172 ceph-mon[25457]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3ddfcd81-94a5-46f7-812e-c87f08345aef"}]': finished 2024-02-07T12:19:32.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:31 smithi172 ceph-mon[25457]: osdmap e5: 1 total, 0 up, 1 in 2024-02-07T12:19:32.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:31 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:19:33.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:32 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2022449117' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:19:33.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:32 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2022449117' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:19:33.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:32 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2022449117' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:19:34.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:33 smithi019 ceph-mon[24574]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:34.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:33 smithi019 ceph-mon[29267]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:34.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:33 smithi172 ceph-mon[25457]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:36.011 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:35 smithi019 ceph-mon[29267]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:36.011 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:35 smithi019 ceph-mon[24574]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:36.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:35 smithi172 ceph-mon[25457]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:38.050 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:37 smithi019 ceph-mon[24574]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:38.050 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:37 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-02-07T12:19:38.050 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:37 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:38.051 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:37 smithi019 ceph-mon[29267]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:38.051 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:37 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-02-07T12:19:38.051 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:37 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:38.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:37 smithi172 ceph-mon[25457]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:38.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:37 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-02-07T12:19:38.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:37 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:39.011 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:38 smithi019 ceph-mon[24574]: Deploying daemon osd.0 on smithi019 2024-02-07T12:19:39.011 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:38 smithi019 ceph-mon[29267]: Deploying daemon osd.0 on smithi019 2024-02-07T12:19:39.020 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:38 smithi172 ceph-mon[25457]: Deploying daemon osd.0 on smithi019 2024-02-07T12:19:40.036 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:39 smithi172 ceph-mon[25457]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:40.038 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:39 smithi019 ceph-mon[24574]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:40.039 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:39 smithi019 ceph-mon[29267]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:42.054 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:41 smithi019 ceph-mon[24574]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:42.054 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:41 smithi019 ceph-mon[29267]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:42.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:41 smithi172 ceph-mon[25457]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:43.581 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:43 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:43.582 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:43 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:43.582 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:43 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:43.582 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:43 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:43.582 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:43 smithi019 ceph-mon[24574]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:43.582 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:43 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:43.582 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:43 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:43.582 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:43 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:43.582 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:43 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:43.582 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:43 smithi019 ceph-mon[29267]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:43.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:43 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:43.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:43 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:43.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:43 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:43.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:43 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:43.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:43 smithi172 ceph-mon[25457]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:45.503 INFO:teuthology.orchestra.run.smithi019.stdout:Created osd(s) 0 on host 'smithi019' 2024-02-07T12:19:45.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:45 smithi019 ceph-mon[24574]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:45.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:45 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:45.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:45 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:45.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:45 smithi019 ceph-mon[24574]: from='osd.0 [v2:172.21.15.19:6802/4112372591,v1:172.21.15.19:6803/4112372591]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-02-07T12:19:45.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:45 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:45.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:45 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:45.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:45 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:45.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:45 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:45.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:45 smithi019 ceph-mon[29267]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:45.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:45 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:45.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:45 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:45.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:45 smithi019 ceph-mon[29267]: from='osd.0 [v2:172.21.15.19:6802/4112372591,v1:172.21.15.19:6803/4112372591]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-02-07T12:19:45.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:45 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:45.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:45 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:45.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:45 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:45.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:45 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:46.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:45 smithi172 ceph-mon[25457]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:46.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:45 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:46.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:45 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:46.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:45 smithi172 ceph-mon[25457]: from='osd.0 [v2:172.21.15.19:6802/4112372591,v1:172.21.15.19:6803/4112372591]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-02-07T12:19:46.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:45 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:46.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:45 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:46.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:45 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:46.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:45 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:46.161 DEBUG:teuthology.orchestra.run.smithi019:osd.0> sudo journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.0.service 2024-02-07T12:19:46.164 INFO:tasks.cephadm:Deploying osd.1 on smithi019 with /dev/vg_nvme/lv_3... 2024-02-07T12:19:46.164 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-02-07T12:19:47.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:47 smithi019 ceph-mon[24574]: from='osd.0 [v2:172.21.15.19:6802/4112372591,v1:172.21.15.19:6803/4112372591]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-02-07T12:19:47.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:47 smithi019 ceph-mon[24574]: osdmap e6: 1 total, 0 up, 1 in 2024-02-07T12:19:47.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:47 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:19:47.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:47 smithi019 ceph-mon[24574]: from='osd.0 [v2:172.21.15.19:6802/4112372591,v1:172.21.15.19:6803/4112372591]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:19:47.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:47 smithi019 ceph-mon[24574]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:47.310 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:47 smithi019 ceph-mon[29267]: from='osd.0 [v2:172.21.15.19:6802/4112372591,v1:172.21.15.19:6803/4112372591]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-02-07T12:19:47.311 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:47 smithi019 ceph-mon[29267]: osdmap e6: 1 total, 0 up, 1 in 2024-02-07T12:19:47.311 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:47 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:19:47.311 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:47 smithi019 ceph-mon[29267]: from='osd.0 [v2:172.21.15.19:6802/4112372591,v1:172.21.15.19:6803/4112372591]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:19:47.311 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:47 smithi019 ceph-mon[29267]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:47.311 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:19:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0[32130]: 2024-02-07T12:19:47.222+0000 7f68fbfa2700 -1 osd.0 0 waiting for initial osdmap 2024-02-07T12:19:47.311 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:19:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0[32130]: 2024-02-07T12:19:47.229+0000 7f68f6138700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:19:47.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:47 smithi172 ceph-mon[25457]: from='osd.0 [v2:172.21.15.19:6802/4112372591,v1:172.21.15.19:6803/4112372591]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-02-07T12:19:47.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:47 smithi172 ceph-mon[25457]: osdmap e6: 1 total, 0 up, 1 in 2024-02-07T12:19:47.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:47 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:19:47.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:47 smithi172 ceph-mon[25457]: from='osd.0 [v2:172.21.15.19:6802/4112372591,v1:172.21.15.19:6803/4112372591]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:19:47.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:47 smithi172 ceph-mon[25457]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:48.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:48 smithi019 ceph-mon[29267]: from='osd.0 [v2:172.21.15.19:6802/4112372591,v1:172.21.15.19:6803/4112372591]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]': finished 2024-02-07T12:19:48.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:48 smithi019 ceph-mon[29267]: osdmap e7: 1 total, 0 up, 1 in 2024-02-07T12:19:48.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:48 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:19:48.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:48 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:19:48.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:48 smithi019 ceph-mon[24574]: from='osd.0 [v2:172.21.15.19:6802/4112372591,v1:172.21.15.19:6803/4112372591]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]': finished 2024-02-07T12:19:48.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:48 smithi019 ceph-mon[24574]: osdmap e7: 1 total, 0 up, 1 in 2024-02-07T12:19:48.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:48 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:19:48.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:48 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:19:48.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:48 smithi172 ceph-mon[25457]: from='osd.0 [v2:172.21.15.19:6802/4112372591,v1:172.21.15.19:6803/4112372591]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]': finished 2024-02-07T12:19:48.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:48 smithi172 ceph-mon[25457]: osdmap e7: 1 total, 0 up, 1 in 2024-02-07T12:19:48.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:48 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:19:48.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:48 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:19:49.577 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:49 smithi019 ceph-mon[24574]: purged_snaps scrub starts 2024-02-07T12:19:49.577 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:49 smithi019 ceph-mon[24574]: purged_snaps scrub ok 2024-02-07T12:19:49.577 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:49 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:19:49.577 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:49 smithi019 ceph-mon[24574]: osd.0 [v2:172.21.15.19:6802/4112372591,v1:172.21.15.19:6803/4112372591] boot 2024-02-07T12:19:49.577 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:49 smithi019 ceph-mon[24574]: osdmap e8: 1 total, 1 up, 1 in 2024-02-07T12:19:49.577 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:49 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:19:49.577 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:49 smithi019 ceph-mon[24574]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:49.578 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:49 smithi019 ceph-mon[29267]: purged_snaps scrub starts 2024-02-07T12:19:49.578 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:49 smithi019 ceph-mon[29267]: purged_snaps scrub ok 2024-02-07T12:19:49.578 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:49 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:19:49.578 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:49 smithi019 ceph-mon[29267]: osd.0 [v2:172.21.15.19:6802/4112372591,v1:172.21.15.19:6803/4112372591] boot 2024-02-07T12:19:49.578 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:49 smithi019 ceph-mon[29267]: osdmap e8: 1 total, 1 up, 1 in 2024-02-07T12:19:49.578 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:49 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:19:49.578 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:49 smithi019 ceph-mon[29267]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:49.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:49 smithi172 ceph-mon[25457]: purged_snaps scrub starts 2024-02-07T12:19:49.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:49 smithi172 ceph-mon[25457]: purged_snaps scrub ok 2024-02-07T12:19:49.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:49 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:19:49.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:49 smithi172 ceph-mon[25457]: osd.0 [v2:172.21.15.19:6802/4112372591,v1:172.21.15.19:6803/4112372591] boot 2024-02-07T12:19:49.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:49 smithi172 ceph-mon[25457]: osdmap e8: 1 total, 1 up, 1 in 2024-02-07T12:19:49.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:49 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:19:49.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:49 smithi172 ceph-mon[25457]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:19:50.094 INFO:teuthology.orchestra.run.smithi019.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-02-07T12:19:50.094 INFO:teuthology.orchestra.run.smithi019.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-02-07T12:19:50.094 INFO:teuthology.orchestra.run.smithi019.stderr: stderr: 10+0 records in 2024-02-07T12:19:50.095 INFO:teuthology.orchestra.run.smithi019.stderr:10+0 records out 2024-02-07T12:19:50.095 INFO:teuthology.orchestra.run.smithi019.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0124091 s, 845 MB/s 2024-02-07T12:19:50.095 INFO:teuthology.orchestra.run.smithi019.stderr:--> Zapping successful for: 2024-02-07T12:19:50.371 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:50 smithi019 ceph-mon[29267]: osdmap e9: 1 total, 1 up, 1 in 2024-02-07T12:19:50.372 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:50 smithi019 ceph-mon[29267]: Detected new or changed devices on smithi019 2024-02-07T12:19:50.372 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:50 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:50.372 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:50 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:19:50.372 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:50 smithi019 ceph-mon[29267]: Adjusting osd_memory_target on smithi019 to 16069M 2024-02-07T12:19:50.372 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:50 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:50.372 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:50 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:50.372 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:50 smithi019 ceph-mon[24574]: osdmap e9: 1 total, 1 up, 1 in 2024-02-07T12:19:50.372 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:50 smithi019 ceph-mon[24574]: Detected new or changed devices on smithi019 2024-02-07T12:19:50.372 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:50 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:50.372 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:50 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:19:50.373 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:50 smithi019 ceph-mon[24574]: Adjusting osd_memory_target on smithi019 to 16069M 2024-02-07T12:19:50.373 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:50 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:50.373 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:50 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:50.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:50 smithi172 ceph-mon[25457]: osdmap e9: 1 total, 1 up, 1 in 2024-02-07T12:19:50.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:50 smithi172 ceph-mon[25457]: Detected new or changed devices on smithi019 2024-02-07T12:19:50.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:50 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:50.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:50 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:19:50.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:50 smithi172 ceph-mon[25457]: Adjusting osd_memory_target on smithi019 to 16069M 2024-02-07T12:19:50.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:50 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:50.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:50 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:19:50.763 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph orch daemon add osd smithi019:vg_nvme/lv_3 2024-02-07T12:19:51.330 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:51 smithi019 ceph-mon[24574]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:19:51.330 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:51 smithi019 ceph-mon[29267]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:19:51.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:51 smithi172 ceph-mon[25457]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:19:52.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:52 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:19:52.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:52 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:19:52.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:52 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:52.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:52 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:19:52.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:52 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:19:52.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:52 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:52.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:52 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:19:52.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:52 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:19:52.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:52 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:53.940 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:53 smithi019 ceph-mon[29267]: from='client.24142 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi019:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:19:53.940 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:53 smithi019 ceph-mon[29267]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:19:53.940 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:53 smithi019 ceph-mon[24574]: from='client.24142 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi019:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:19:53.940 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:53 smithi019 ceph-mon[24574]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:19:54.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:53 smithi172 ceph-mon[25457]: from='client.24142 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi019:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:19:54.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:53 smithi172 ceph-mon[25457]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:19:56.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:55 smithi019 ceph-mon[24574]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:19:56.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:55 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/4069091868' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c5266822-bc44-4ede-90a8-bc3f880c1ea8"}]: dispatch 2024-02-07T12:19:56.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:55 smithi019 ceph-mon[24574]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c5266822-bc44-4ede-90a8-bc3f880c1ea8"}]: dispatch 2024-02-07T12:19:56.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:55 smithi019 ceph-mon[24574]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c5266822-bc44-4ede-90a8-bc3f880c1ea8"}]': finished 2024-02-07T12:19:56.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:55 smithi019 ceph-mon[24574]: osdmap e10: 2 total, 1 up, 2 in 2024-02-07T12:19:56.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:55 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:19:56.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:55 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3264052220' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:19:56.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:55 smithi019 ceph-mon[29267]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:19:56.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:55 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/4069091868' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c5266822-bc44-4ede-90a8-bc3f880c1ea8"}]: dispatch 2024-02-07T12:19:56.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:55 smithi019 ceph-mon[29267]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c5266822-bc44-4ede-90a8-bc3f880c1ea8"}]: dispatch 2024-02-07T12:19:56.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:55 smithi019 ceph-mon[29267]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c5266822-bc44-4ede-90a8-bc3f880c1ea8"}]': finished 2024-02-07T12:19:56.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:55 smithi019 ceph-mon[29267]: osdmap e10: 2 total, 1 up, 2 in 2024-02-07T12:19:56.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:55 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:19:56.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:55 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3264052220' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:19:56.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:55 smithi172 ceph-mon[25457]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:19:56.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:55 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/4069091868' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c5266822-bc44-4ede-90a8-bc3f880c1ea8"}]: dispatch 2024-02-07T12:19:56.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:55 smithi172 ceph-mon[25457]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c5266822-bc44-4ede-90a8-bc3f880c1ea8"}]: dispatch 2024-02-07T12:19:56.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:55 smithi172 ceph-mon[25457]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c5266822-bc44-4ede-90a8-bc3f880c1ea8"}]': finished 2024-02-07T12:19:56.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:55 smithi172 ceph-mon[25457]: osdmap e10: 2 total, 1 up, 2 in 2024-02-07T12:19:56.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:55 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:19:56.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:55 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3264052220' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:19:57.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:57 smithi019 ceph-mon[24574]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:19:57.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:57 smithi019 ceph-mon[29267]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:19:58.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:57 smithi172 ceph-mon[25457]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:19:59.723 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:19:59 smithi019 ceph-mon[24574]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:19:59.724 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:19:59 smithi019 ceph-mon[29267]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:20:00.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:19:59 smithi172 ceph-mon[25457]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:20:00.827 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:00 smithi019 ceph-mon[24574]: overall HEALTH_OK 2024-02-07T12:20:00.828 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:00 smithi019 ceph-mon[29267]: overall HEALTH_OK 2024-02-07T12:20:01.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:00 smithi172 ceph-mon[25457]: overall HEALTH_OK 2024-02-07T12:20:01.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:01 smithi019 ceph-mon[24574]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:20:01.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:01 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-02-07T12:20:01.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:01 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:01.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:01 smithi019 ceph-mon[24574]: Deploying daemon osd.1 on smithi019 2024-02-07T12:20:01.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:01 smithi019 ceph-mon[29267]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:20:01.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:01 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-02-07T12:20:01.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:01 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:01.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:01 smithi019 ceph-mon[29267]: Deploying daemon osd.1 on smithi019 2024-02-07T12:20:02.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:01 smithi172 ceph-mon[25457]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:20:02.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:01 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-02-07T12:20:02.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:01 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:02.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:01 smithi172 ceph-mon[25457]: Deploying daemon osd.1 on smithi019 2024-02-07T12:20:03.983 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:03 smithi019 ceph-mon[24574]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:20:03.983 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:03 smithi019 ceph-mon[29267]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:20:04.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:03 smithi172 ceph-mon[25457]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:20:05.883 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:05 smithi019 ceph-mon[24574]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:20:05.883 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:05 smithi019 ceph-mon[29267]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:20:05.912 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:05 smithi172 ceph-mon[25457]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:20:07.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:06 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:07.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:06 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:20:07.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:06 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:07.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:06 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:20:07.135 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:06 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:07.135 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:06 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:20:07.135 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:06 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:07.135 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:06 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:20:07.136 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:06 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:07.136 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:06 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:20:07.136 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:06 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:07.136 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:06 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:20:08.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:07 smithi019 ceph-mon[24574]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:20:08.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:07 smithi019 ceph-mon[29267]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:20:08.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:07 smithi172 ceph-mon[25457]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:20:08.853 INFO:teuthology.orchestra.run.smithi019.stdout:Created osd(s) 1 on host 'smithi019' 2024-02-07T12:20:09.464 DEBUG:teuthology.orchestra.run.smithi019:osd.1> sudo journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.1.service 2024-02-07T12:20:09.466 INFO:tasks.cephadm:Deploying osd.2 on smithi019 with /dev/vg_nvme/lv_2... 2024-02-07T12:20:09.467 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-02-07T12:20:10.051 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:09 smithi019 ceph-mon[24574]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:20:10.052 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:09 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:10.052 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:09 smithi019 ceph-mon[24574]: from='osd.1 [v2:172.21.15.19:6810/3625191644,v1:172.21.15.19:6811/3625191644]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-02-07T12:20:10.052 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:09 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:10.052 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:09 smithi019 ceph-mon[29267]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:20:10.052 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:09 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:10.052 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:09 smithi019 ceph-mon[29267]: from='osd.1 [v2:172.21.15.19:6810/3625191644,v1:172.21.15.19:6811/3625191644]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-02-07T12:20:10.052 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:09 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:10.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:09 smithi172 ceph-mon[25457]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:20:10.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:09 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:10.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:09 smithi172 ceph-mon[25457]: from='osd.1 [v2:172.21.15.19:6810/3625191644,v1:172.21.15.19:6811/3625191644]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-02-07T12:20:10.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:09 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:11.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:10 smithi019 ceph-mon[24574]: from='osd.1 [v2:172.21.15.19:6810/3625191644,v1:172.21.15.19:6811/3625191644]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-02-07T12:20:11.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:10 smithi019 ceph-mon[24574]: osdmap e11: 2 total, 1 up, 2 in 2024-02-07T12:20:11.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:10 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:20:11.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:10 smithi019 ceph-mon[24574]: from='osd.1 [v2:172.21.15.19:6810/3625191644,v1:172.21.15.19:6811/3625191644]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:20:11.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:10 smithi019 ceph-mon[29267]: from='osd.1 [v2:172.21.15.19:6810/3625191644,v1:172.21.15.19:6811/3625191644]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-02-07T12:20:11.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:10 smithi019 ceph-mon[29267]: osdmap e11: 2 total, 1 up, 2 in 2024-02-07T12:20:11.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:10 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:20:11.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:10 smithi019 ceph-mon[29267]: from='osd.1 [v2:172.21.15.19:6810/3625191644,v1:172.21.15.19:6811/3625191644]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:20:11.096 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:20:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1[35085]: 2024-02-07T12:20:10.865+0000 7f71a9ece700 -1 osd.1 0 waiting for initial osdmap 2024-02-07T12:20:11.096 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:20:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1[35085]: 2024-02-07T12:20:10.868+0000 7f71a4064700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:20:11.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:10 smithi172 ceph-mon[25457]: from='osd.1 [v2:172.21.15.19:6810/3625191644,v1:172.21.15.19:6811/3625191644]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-02-07T12:20:11.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:10 smithi172 ceph-mon[25457]: osdmap e11: 2 total, 1 up, 2 in 2024-02-07T12:20:11.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:10 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:20:11.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:10 smithi172 ceph-mon[25457]: from='osd.1 [v2:172.21.15.19:6810/3625191644,v1:172.21.15.19:6811/3625191644]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:20:11.384 INFO:teuthology.orchestra.run.smithi019.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-02-07T12:20:11.384 INFO:teuthology.orchestra.run.smithi019.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-02-07T12:20:11.384 INFO:teuthology.orchestra.run.smithi019.stderr: stderr: 10+0 records in 2024-02-07T12:20:11.385 INFO:teuthology.orchestra.run.smithi019.stderr:10+0 records out 2024-02-07T12:20:11.385 INFO:teuthology.orchestra.run.smithi019.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.012176 s, 861 MB/s 2024-02-07T12:20:11.385 INFO:teuthology.orchestra.run.smithi019.stderr:--> Zapping successful for: 2024-02-07T12:20:12.064 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:11 smithi019 ceph-mon[29267]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:20:12.064 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:11 smithi019 ceph-mon[29267]: from='osd.1 [v2:172.21.15.19:6810/3625191644,v1:172.21.15.19:6811/3625191644]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]': finished 2024-02-07T12:20:12.064 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:11 smithi019 ceph-mon[29267]: osdmap e12: 2 total, 1 up, 2 in 2024-02-07T12:20:12.064 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:11 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:20:12.064 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:11 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:20:12.064 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:11 smithi019 ceph-mon[24574]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:20:12.064 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:11 smithi019 ceph-mon[24574]: from='osd.1 [v2:172.21.15.19:6810/3625191644,v1:172.21.15.19:6811/3625191644]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]': finished 2024-02-07T12:20:12.064 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:11 smithi019 ceph-mon[24574]: osdmap e12: 2 total, 1 up, 2 in 2024-02-07T12:20:12.064 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:11 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:20:12.065 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:11 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:20:12.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:11 smithi172 ceph-mon[25457]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:20:12.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:11 smithi172 ceph-mon[25457]: from='osd.1 [v2:172.21.15.19:6810/3625191644,v1:172.21.15.19:6811/3625191644]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]': finished 2024-02-07T12:20:12.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:11 smithi172 ceph-mon[25457]: osdmap e12: 2 total, 1 up, 2 in 2024-02-07T12:20:12.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:11 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:20:12.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:11 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:20:12.970 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:12 smithi019 ceph-mon[24574]: purged_snaps scrub starts 2024-02-07T12:20:12.970 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:12 smithi019 ceph-mon[24574]: purged_snaps scrub ok 2024-02-07T12:20:12.970 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:12 smithi019 ceph-mon[24574]: osd.1 [v2:172.21.15.19:6810/3625191644,v1:172.21.15.19:6811/3625191644] boot 2024-02-07T12:20:12.970 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:12 smithi019 ceph-mon[24574]: osdmap e13: 2 total, 2 up, 2 in 2024-02-07T12:20:12.970 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:12 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:20:12.970 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:12 smithi019 ceph-mon[29267]: purged_snaps scrub starts 2024-02-07T12:20:12.970 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:12 smithi019 ceph-mon[29267]: purged_snaps scrub ok 2024-02-07T12:20:12.970 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:12 smithi019 ceph-mon[29267]: osd.1 [v2:172.21.15.19:6810/3625191644,v1:172.21.15.19:6811/3625191644] boot 2024-02-07T12:20:12.970 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:12 smithi019 ceph-mon[29267]: osdmap e13: 2 total, 2 up, 2 in 2024-02-07T12:20:12.971 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:12 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:20:13.301 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph orch daemon add osd smithi019:vg_nvme/lv_2 2024-02-07T12:20:13.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:12 smithi172 ceph-mon[25457]: purged_snaps scrub starts 2024-02-07T12:20:13.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:12 smithi172 ceph-mon[25457]: purged_snaps scrub ok 2024-02-07T12:20:13.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:12 smithi172 ceph-mon[25457]: osd.1 [v2:172.21.15.19:6810/3625191644,v1:172.21.15.19:6811/3625191644] boot 2024-02-07T12:20:13.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:12 smithi172 ceph-mon[25457]: osdmap e13: 2 total, 2 up, 2 in 2024-02-07T12:20:13.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:12 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:20:14.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:13 smithi019 ceph-mon[24574]: pgmap v55: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:20:14.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:13 smithi019 ceph-mon[24574]: osdmap e14: 2 total, 2 up, 2 in 2024-02-07T12:20:14.042 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:13 smithi019 ceph-mon[29267]: pgmap v55: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:20:14.043 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:13 smithi019 ceph-mon[29267]: osdmap e14: 2 total, 2 up, 2 in 2024-02-07T12:20:14.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:13 smithi172 ceph-mon[25457]: pgmap v55: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:20:14.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:13 smithi172 ceph-mon[25457]: osdmap e14: 2 total, 2 up, 2 in 2024-02-07T12:20:15.146 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:15 smithi019 ceph-mon[24574]: Detected new or changed devices on smithi019 2024-02-07T12:20:15.146 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:15 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:15.146 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:15 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:20:15.146 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:15 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:20:15.146 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:15 smithi019 ceph-mon[24574]: Adjusting osd_memory_target on smithi019 to 8034M 2024-02-07T12:20:15.146 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:15 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:15.146 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:15 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:15.146 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:15 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:20:15.146 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:15 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:15.147 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:15 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:20:15.147 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:15 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:15.147 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:15 smithi019 ceph-mon[24574]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:15.147 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:15 smithi019 ceph-mon[29267]: Detected new or changed devices on smithi019 2024-02-07T12:20:15.147 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:15 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:15.147 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:15 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:20:15.147 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:15 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:20:15.147 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:15 smithi019 ceph-mon[29267]: Adjusting osd_memory_target on smithi019 to 8034M 2024-02-07T12:20:15.147 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:15 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:15.147 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:15 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:15.147 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:15 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:20:15.148 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:15 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:15.148 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:15 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:20:15.148 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:15 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:15.148 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:15 smithi019 ceph-mon[29267]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:15.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:15 smithi172 ceph-mon[25457]: Detected new or changed devices on smithi019 2024-02-07T12:20:15.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:15 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:15.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:15 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:20:15.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:15 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:20:15.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:15 smithi172 ceph-mon[25457]: Adjusting osd_memory_target on smithi019 to 8034M 2024-02-07T12:20:15.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:15 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:15.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:15 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:15.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:15 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:20:15.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:15 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:15.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:15 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:20:15.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:15 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:15.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:15 smithi172 ceph-mon[25457]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:16.234 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:16 smithi019 ceph-mon[29267]: from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi019:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:20:16.234 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:16 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:20:16.234 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:16 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:20:16.234 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:16 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:16.235 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:16 smithi019 ceph-mon[24574]: from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi019:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:20:16.235 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:16 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:20:16.235 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:16 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:20:16.235 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:16 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:16.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:16 smithi172 ceph-mon[25457]: from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi019:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:20:16.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:16 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:20:16.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:16 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:20:16.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:16 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:17.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:17 smithi019 ceph-mon[24574]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:17.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:17 smithi019 ceph-mon[29267]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:17.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:17 smithi172 ceph-mon[25457]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:19.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:18 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3505354696' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1df65fd6-bed8-4ada-ba4c-7cde82b45a31"}]: dispatch 2024-02-07T12:20:19.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:18 smithi019 ceph-mon[24574]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1df65fd6-bed8-4ada-ba4c-7cde82b45a31"}]: dispatch 2024-02-07T12:20:19.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:18 smithi019 ceph-mon[24574]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1df65fd6-bed8-4ada-ba4c-7cde82b45a31"}]': finished 2024-02-07T12:20:19.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:18 smithi019 ceph-mon[24574]: osdmap e15: 3 total, 2 up, 3 in 2024-02-07T12:20:19.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:18 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:20:19.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:18 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2539256064' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:20:19.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:18 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3505354696' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1df65fd6-bed8-4ada-ba4c-7cde82b45a31"}]: dispatch 2024-02-07T12:20:19.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:18 smithi019 ceph-mon[29267]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1df65fd6-bed8-4ada-ba4c-7cde82b45a31"}]: dispatch 2024-02-07T12:20:19.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:18 smithi019 ceph-mon[29267]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1df65fd6-bed8-4ada-ba4c-7cde82b45a31"}]': finished 2024-02-07T12:20:19.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:18 smithi019 ceph-mon[29267]: osdmap e15: 3 total, 2 up, 3 in 2024-02-07T12:20:19.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:18 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:20:19.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:18 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2539256064' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:20:19.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:18 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3505354696' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1df65fd6-bed8-4ada-ba4c-7cde82b45a31"}]: dispatch 2024-02-07T12:20:19.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:18 smithi172 ceph-mon[25457]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1df65fd6-bed8-4ada-ba4c-7cde82b45a31"}]: dispatch 2024-02-07T12:20:19.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:18 smithi172 ceph-mon[25457]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1df65fd6-bed8-4ada-ba4c-7cde82b45a31"}]': finished 2024-02-07T12:20:19.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:18 smithi172 ceph-mon[25457]: osdmap e15: 3 total, 2 up, 3 in 2024-02-07T12:20:19.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:18 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:20:19.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:18 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2539256064' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:20:20.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:19 smithi019 ceph-mon[24574]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:20.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:19 smithi019 ceph-mon[29267]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:20.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:19 smithi172 ceph-mon[25457]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:21.967 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:21 smithi019 ceph-mon[29267]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:21.968 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:21 smithi019 ceph-mon[24574]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:22.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:21 smithi172 ceph-mon[25457]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:24.092 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:23 smithi019 ceph-mon[29267]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:24.093 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:23 smithi019 ceph-mon[24574]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:24.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:23 smithi172 ceph-mon[25457]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:24.828 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:24 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-02-07T12:20:24.828 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:24 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:24.829 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:24 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-02-07T12:20:24.829 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:24 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:25.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:24 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-02-07T12:20:25.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:24 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:25.727 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:25 smithi019 ceph-mon[29267]: Deploying daemon osd.2 on smithi019 2024-02-07T12:20:25.727 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:25 smithi019 ceph-mon[29267]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:25.727 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:25 smithi019 ceph-mon[24574]: Deploying daemon osd.2 on smithi019 2024-02-07T12:20:25.727 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:25 smithi019 ceph-mon[24574]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:26.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:25 smithi172 ceph-mon[25457]: Deploying daemon osd.2 on smithi019 2024-02-07T12:20:26.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:25 smithi172 ceph-mon[25457]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:28.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:27 smithi019 ceph-mon[24574]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:28.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:27 smithi019 ceph-mon[29267]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:28.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:27 smithi172 ceph-mon[25457]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:29.727 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:29 smithi019 ceph-mon[29267]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:29.727 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:29 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:20:29.727 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:29 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:20:29.727 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:29 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:29.727 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:29 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:20:29.727 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:29 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:29.727 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:29 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:20:29.728 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:29 smithi019 ceph-mon[24574]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:29.728 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:29 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:20:29.728 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:29 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:20:29.728 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:29 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:29.728 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:29 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:20:29.728 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:29 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:29.728 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:29 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:20:30.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:29 smithi172 ceph-mon[25457]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:30.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:29 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:20:30.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:29 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:20:30.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:29 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:30.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:29 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:20:30.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:29 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:30.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:29 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:20:32.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:31 smithi019 ceph-mon[24574]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:32.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:31 smithi019 ceph-mon[29267]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:32.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:31 smithi172 ceph-mon[25457]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:32.186 INFO:teuthology.orchestra.run.smithi019.stdout:Created osd(s) 2 on host 'smithi019' 2024-02-07T12:20:32.792 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:32 smithi019 ceph-mon[29267]: from='osd.2 [v2:172.21.15.19:6818/4106092366,v1:172.21.15.19:6819/4106092366]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-02-07T12:20:32.792 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:32 smithi019 ceph-mon[29267]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-02-07T12:20:32.792 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:32 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:32.792 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:32 smithi019 ceph-mon[24574]: from='osd.2 [v2:172.21.15.19:6818/4106092366,v1:172.21.15.19:6819/4106092366]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-02-07T12:20:32.792 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:32 smithi019 ceph-mon[24574]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-02-07T12:20:32.792 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:32 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:32.817 DEBUG:teuthology.orchestra.run.smithi019:osd.2> sudo journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.2.service 2024-02-07T12:20:32.820 INFO:tasks.cephadm:Deploying osd.3 on smithi019 with /dev/vg_nvme/lv_1... 2024-02-07T12:20:32.820 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-02-07T12:20:33.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:32 smithi172 ceph-mon[25457]: from='osd.2 [v2:172.21.15.19:6818/4106092366,v1:172.21.15.19:6819/4106092366]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-02-07T12:20:33.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:32 smithi172 ceph-mon[25457]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-02-07T12:20:33.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:32 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:33.982 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:33 smithi019 ceph-mon[24574]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:33.983 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:33 smithi019 ceph-mon[24574]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-02-07T12:20:33.983 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:33 smithi019 ceph-mon[24574]: osdmap e16: 3 total, 2 up, 3 in 2024-02-07T12:20:33.983 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:33 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:20:33.983 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:33 smithi019 ceph-mon[24574]: from='osd.2 [v2:172.21.15.19:6818/4106092366,v1:172.21.15.19:6819/4106092366]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:20:33.983 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:33 smithi019 ceph-mon[24574]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:20:33.983 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:33 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:33.983 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:33 smithi019 ceph-mon[29267]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:33.983 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:33 smithi019 ceph-mon[29267]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-02-07T12:20:33.983 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:33 smithi019 ceph-mon[29267]: osdmap e16: 3 total, 2 up, 3 in 2024-02-07T12:20:33.984 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:33 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:20:33.984 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:33 smithi019 ceph-mon[29267]: from='osd.2 [v2:172.21.15.19:6818/4106092366,v1:172.21.15.19:6819/4106092366]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:20:33.984 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:33 smithi019 ceph-mon[29267]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:20:33.984 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:33 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:33.984 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:20:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2[38018]: 2024-02-07T12:20:33.680+0000 7f54d6e77700 -1 osd.2 0 waiting for initial osdmap 2024-02-07T12:20:33.984 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:20:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2[38018]: 2024-02-07T12:20:33.683+0000 7f54d3812700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:20:34.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:33 smithi172 ceph-mon[25457]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:34.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:33 smithi172 ceph-mon[25457]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-02-07T12:20:34.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:33 smithi172 ceph-mon[25457]: osdmap e16: 3 total, 2 up, 3 in 2024-02-07T12:20:34.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:33 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:20:34.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:33 smithi172 ceph-mon[25457]: from='osd.2 [v2:172.21.15.19:6818/4106092366,v1:172.21.15.19:6819/4106092366]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:20:34.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:33 smithi172 ceph-mon[25457]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:20:34.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:33 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:34.880 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:34 smithi019 ceph-mon[29267]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]': finished 2024-02-07T12:20:34.880 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:34 smithi019 ceph-mon[29267]: osdmap e17: 3 total, 2 up, 3 in 2024-02-07T12:20:34.880 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:34 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:20:34.880 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:34 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:20:34.880 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:34 smithi019 ceph-mon[24574]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]': finished 2024-02-07T12:20:34.880 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:34 smithi019 ceph-mon[24574]: osdmap e17: 3 total, 2 up, 3 in 2024-02-07T12:20:34.880 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:34 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:20:34.880 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:34 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:20:35.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:34 smithi172 ceph-mon[25457]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]': finished 2024-02-07T12:20:35.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:34 smithi172 ceph-mon[25457]: osdmap e17: 3 total, 2 up, 3 in 2024-02-07T12:20:35.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:34 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:20:35.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:34 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:20:35.778 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:35 smithi019 ceph-mon[29267]: purged_snaps scrub starts 2024-02-07T12:20:35.778 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:35 smithi019 ceph-mon[29267]: purged_snaps scrub ok 2024-02-07T12:20:35.778 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:35 smithi019 ceph-mon[29267]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:35.778 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:35 smithi019 ceph-mon[29267]: osd.2 [v2:172.21.15.19:6818/4106092366,v1:172.21.15.19:6819/4106092366] boot 2024-02-07T12:20:35.778 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:35 smithi019 ceph-mon[29267]: osdmap e18: 3 total, 3 up, 3 in 2024-02-07T12:20:35.779 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:35 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:20:35.779 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:35 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-02-07T12:20:35.779 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:35 smithi019 ceph-mon[24574]: purged_snaps scrub starts 2024-02-07T12:20:35.779 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:35 smithi019 ceph-mon[24574]: purged_snaps scrub ok 2024-02-07T12:20:35.779 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:35 smithi019 ceph-mon[24574]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:35.779 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:35 smithi019 ceph-mon[24574]: osd.2 [v2:172.21.15.19:6818/4106092366,v1:172.21.15.19:6819/4106092366] boot 2024-02-07T12:20:35.779 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:35 smithi019 ceph-mon[24574]: osdmap e18: 3 total, 3 up, 3 in 2024-02-07T12:20:35.779 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:35 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:20:35.779 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:35 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-02-07T12:20:36.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:35 smithi172 ceph-mon[25457]: purged_snaps scrub starts 2024-02-07T12:20:36.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:35 smithi172 ceph-mon[25457]: purged_snaps scrub ok 2024-02-07T12:20:36.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:35 smithi172 ceph-mon[25457]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:20:36.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:35 smithi172 ceph-mon[25457]: osd.2 [v2:172.21.15.19:6818/4106092366,v1:172.21.15.19:6819/4106092366] boot 2024-02-07T12:20:36.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:35 smithi172 ceph-mon[25457]: osdmap e18: 3 total, 3 up, 3 in 2024-02-07T12:20:36.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:35 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:20:36.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:35 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-02-07T12:20:36.469 INFO:teuthology.orchestra.run.smithi019.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-02-07T12:20:36.470 INFO:teuthology.orchestra.run.smithi019.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-02-07T12:20:36.470 INFO:teuthology.orchestra.run.smithi019.stderr: stderr: 10+0 records in 2024-02-07T12:20:36.470 INFO:teuthology.orchestra.run.smithi019.stderr:10+0 records out 2024-02-07T12:20:36.470 INFO:teuthology.orchestra.run.smithi019.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0138784 s, 756 MB/s 2024-02-07T12:20:36.470 INFO:teuthology.orchestra.run.smithi019.stderr:--> Zapping successful for: 2024-02-07T12:20:36.909 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:36 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-02-07T12:20:36.909 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:36 smithi019 ceph-mon[24574]: osdmap e19: 3 total, 3 up, 3 in 2024-02-07T12:20:36.909 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:36 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-02-07T12:20:36.909 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:36 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:36.909 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:36 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:20:36.910 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:36 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:20:36.910 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:36 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:20:36.910 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:36 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:36.910 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:36 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:36.910 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:36 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:20:36.910 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:36 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:36.910 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:36 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:20:36.910 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:36 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:36.910 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:36 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-02-07T12:20:36.910 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:36 smithi019 ceph-mon[29267]: osdmap e19: 3 total, 3 up, 3 in 2024-02-07T12:20:36.910 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:36 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-02-07T12:20:36.910 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:36 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:36.910 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:36 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:20:36.911 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:36 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:20:36.911 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:36 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:20:36.911 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:36 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:36.911 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:36 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:36.911 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:36 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:20:36.911 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:36 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:36.911 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:36 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:20:36.911 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:36 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:37.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:36 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-02-07T12:20:37.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:36 smithi172 ceph-mon[25457]: osdmap e19: 3 total, 3 up, 3 in 2024-02-07T12:20:37.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:36 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-02-07T12:20:37.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:36 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:37.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:36 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:20:37.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:36 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:20:37.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:36 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:20:37.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:36 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:37.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:36 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:37.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:36 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:20:37.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:36 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:37.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:36 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:20:37.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:36 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:37.172 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:20:37 smithi019 sudo[39233]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-02-07T12:20:37.172 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:20:37 smithi019 sudo[39233]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-02-07T12:20:37.172 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:20:37 smithi019 sudo[39233]: pam_unix(sudo:session): session closed for user root 2024-02-07T12:20:37.173 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph orch daemon add osd smithi019:vg_nvme/lv_1 2024-02-07T12:20:37.494 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:20:37 smithi019 sudo[39254]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-02-07T12:20:37.494 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:20:37 smithi019 sudo[39254]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-02-07T12:20:37.494 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:20:37 smithi019 sudo[39254]: pam_unix(sudo:session): session closed for user root 2024-02-07T12:20:37.770 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:37 smithi019 ceph-mon[29267]: Detected new or changed devices on smithi019 2024-02-07T12:20:37.771 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:37 smithi019 ceph-mon[29267]: Adjusting osd_memory_target on smithi019 to 5356M 2024-02-07T12:20:37.771 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:37 smithi019 ceph-mon[29267]: pgmap v73: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:20:37.771 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:37 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-02-07T12:20:37.771 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:37 smithi019 ceph-mon[29267]: osdmap e20: 3 total, 3 up, 3 in 2024-02-07T12:20:37.771 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:37 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:37.771 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:20:37 smithi019 sudo[39284]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-02-07T12:20:37.771 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:20:37 smithi019 sudo[39284]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-02-07T12:20:37.771 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:20:37 smithi019 sudo[39284]: pam_unix(sudo:session): session closed for user root 2024-02-07T12:20:37.772 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:37 smithi019 ceph-mon[24574]: Detected new or changed devices on smithi019 2024-02-07T12:20:37.772 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:37 smithi019 ceph-mon[24574]: Adjusting osd_memory_target on smithi019 to 5356M 2024-02-07T12:20:37.772 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:37 smithi019 ceph-mon[24574]: pgmap v73: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:20:37.772 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:37 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-02-07T12:20:37.772 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:37 smithi019 ceph-mon[24574]: osdmap e20: 3 total, 3 up, 3 in 2024-02-07T12:20:37.772 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:37 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:38.023 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:20:37 smithi019 sudo[39304]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-02-07T12:20:38.023 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:20:37 smithi019 sudo[39304]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-02-07T12:20:38.023 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:20:37 smithi019 sudo[39304]: pam_unix(sudo:session): session closed for user root 2024-02-07T12:20:38.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:37 smithi172 ceph-mon[25457]: Detected new or changed devices on smithi019 2024-02-07T12:20:38.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:37 smithi172 ceph-mon[25457]: Adjusting osd_memory_target on smithi019 to 5356M 2024-02-07T12:20:38.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:37 smithi172 ceph-mon[25457]: pgmap v73: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:20:38.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:37 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-02-07T12:20:38.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:37 smithi172 ceph-mon[25457]: osdmap e20: 3 total, 3 up, 3 in 2024-02-07T12:20:38.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:37 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:38.298 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:20:38 smithi019 sudo[39309]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-02-07T12:20:38.298 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:20:38 smithi019 sudo[39309]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-02-07T12:20:38.298 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:20:38 smithi019 sudo[39309]: pam_unix(sudo:session): session closed for user root 2024-02-07T12:20:38.594 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:20:38 smithi019 sudo[39330]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-02-07T12:20:38.594 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:20:38 smithi019 sudo[39330]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-02-07T12:20:38.594 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:20:38 smithi019 sudo[39330]: pam_unix(sudo:session): session closed for user root 2024-02-07T12:20:38.909 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:38 smithi019 sudo[39340]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-02-07T12:20:38.909 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:38 smithi019 sudo[39340]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-02-07T12:20:38.909 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:38 smithi019 sudo[39340]: pam_unix(sudo:session): session closed for user root 2024-02-07T12:20:39.184 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:38 smithi019 sudo[39363]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-02-07T12:20:39.185 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:38 smithi019 sudo[39363]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-02-07T12:20:39.185 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:38 smithi019 sudo[39363]: pam_unix(sudo:session): session closed for user root 2024-02-07T12:20:39.185 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:38 smithi019 ceph-mon[24574]: osdmap e21: 3 total, 3 up, 3 in 2024-02-07T12:20:39.185 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:38 smithi019 ceph-mon[24574]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-02-07T12:20:39.185 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:38 smithi019 ceph-mon[29267]: osdmap e21: 3 total, 3 up, 3 in 2024-02-07T12:20:39.185 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:38 smithi019 ceph-mon[29267]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-02-07T12:20:39.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:38 smithi172 ceph-mon[25457]: osdmap e21: 3 total, 3 up, 3 in 2024-02-07T12:20:39.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:38 smithi172 ceph-mon[25457]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-02-07T12:20:39.436 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:39 smithi019 sudo[39424]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-02-07T12:20:39.436 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:39 smithi019 sudo[39424]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-02-07T12:20:39.436 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:39 smithi019 sudo[39424]: pam_unix(sudo:session): session closed for user root 2024-02-07T12:20:39.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:39 smithi019 sudo[39453]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-02-07T12:20:39.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:39 smithi019 sudo[39453]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-02-07T12:20:39.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:39 smithi019 sudo[39453]: pam_unix(sudo:session): session closed for user root 2024-02-07T12:20:39.949 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:39 smithi172 sudo[26969]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-02-07T12:20:39.949 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:39 smithi172 sudo[26969]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-02-07T12:20:39.949 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:39 smithi172 sudo[26969]: pam_unix(sudo:session): session closed for user root 2024-02-07T12:20:39.949 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:39 smithi172 sudo[26972]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-02-07T12:20:40.258 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[29267]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:20:40.258 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[29267]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-02-07T12:20:40.258 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:20:40.258 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:20:40.258 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:20:40.258 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[29267]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-02-07T12:20:40.258 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:20:40.259 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:20:40.259 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:20:40.259 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[29267]: from='client.24202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi019:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:20:40.259 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:20:40.259 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:20:40.259 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:40.259 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[29267]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-02-07T12:20:40.259 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[29267]: mgrmap e15: y(active, since 2m), standbys: x 2024-02-07T12:20:40.259 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:20:40.259 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:20:40.259 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:20:40.259 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[24574]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:20:40.260 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[24574]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-02-07T12:20:40.260 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:20:40.260 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:20:40.260 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:20:40.260 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[24574]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-02-07T12:20:40.260 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:20:40.260 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:20:40.260 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:20:40.260 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[24574]: from='client.24202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi019:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:20:40.260 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:20:40.260 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:20:40.260 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:40.260 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[24574]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-02-07T12:20:40.261 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[24574]: mgrmap e15: y(active, since 2m), standbys: x 2024-02-07T12:20:40.261 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:20:40.261 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:20:40.261 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:39 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:20:40.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:39 smithi172 sudo[26972]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-02-07T12:20:40.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:39 smithi172 sudo[26972]: pam_unix(sudo:session): session closed for user root 2024-02-07T12:20:40.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:39 smithi172 ceph-mon[25457]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:20:40.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:39 smithi172 ceph-mon[25457]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-02-07T12:20:40.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:39 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:20:40.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:39 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:20:40.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:39 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:20:40.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:39 smithi172 ceph-mon[25457]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-02-07T12:20:40.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:39 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:20:40.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:39 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:20:40.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:39 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:20:40.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:39 smithi172 ceph-mon[25457]: from='client.24202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi019:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:20:40.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:39 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:20:40.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:39 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:20:40.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:39 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:40.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:39 smithi172 ceph-mon[25457]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-02-07T12:20:40.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:39 smithi172 ceph-mon[25457]: mgrmap e15: y(active, since 2m), standbys: x 2024-02-07T12:20:40.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:39 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:20:40.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:39 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:20:40.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:39 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:20:41.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:40 smithi019 ceph-mon[24574]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-02-07T12:20:41.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:40 smithi019 ceph-mon[24574]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-02-07T12:20:41.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:40 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:41.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:40 smithi019 ceph-mon[24574]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:20:41.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:40 smithi019 ceph-mon[29267]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-02-07T12:20:41.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:40 smithi019 ceph-mon[29267]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-02-07T12:20:41.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:40 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:41.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:40 smithi019 ceph-mon[29267]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:20:41.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:40 smithi172 ceph-mon[25457]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-02-07T12:20:41.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:40 smithi172 ceph-mon[25457]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-02-07T12:20:41.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:40 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:41.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:40 smithi172 ceph-mon[25457]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:20:42.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:41 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1570766091' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "749f4475-2dbd-4166-b49b-3eaf63e1f7bd"}]: dispatch 2024-02-07T12:20:42.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:41 smithi019 ceph-mon[24574]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "749f4475-2dbd-4166-b49b-3eaf63e1f7bd"}]: dispatch 2024-02-07T12:20:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:41 smithi019 ceph-mon[24574]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "749f4475-2dbd-4166-b49b-3eaf63e1f7bd"}]': finished 2024-02-07T12:20:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:41 smithi019 ceph-mon[24574]: osdmap e22: 4 total, 3 up, 4 in 2024-02-07T12:20:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:41 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:20:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:41 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2393344352' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:20:42.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:41 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1570766091' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "749f4475-2dbd-4166-b49b-3eaf63e1f7bd"}]: dispatch 2024-02-07T12:20:42.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:41 smithi019 ceph-mon[29267]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "749f4475-2dbd-4166-b49b-3eaf63e1f7bd"}]: dispatch 2024-02-07T12:20:42.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:41 smithi019 ceph-mon[29267]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "749f4475-2dbd-4166-b49b-3eaf63e1f7bd"}]': finished 2024-02-07T12:20:42.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:41 smithi019 ceph-mon[29267]: osdmap e22: 4 total, 3 up, 4 in 2024-02-07T12:20:42.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:41 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:20:42.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:41 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2393344352' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:20:42.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:41 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1570766091' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "749f4475-2dbd-4166-b49b-3eaf63e1f7bd"}]: dispatch 2024-02-07T12:20:42.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:41 smithi172 ceph-mon[25457]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "749f4475-2dbd-4166-b49b-3eaf63e1f7bd"}]: dispatch 2024-02-07T12:20:42.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:41 smithi172 ceph-mon[25457]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "749f4475-2dbd-4166-b49b-3eaf63e1f7bd"}]': finished 2024-02-07T12:20:42.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:41 smithi172 ceph-mon[25457]: osdmap e22: 4 total, 3 up, 4 in 2024-02-07T12:20:42.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:41 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:20:42.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:41 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2393344352' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:20:43.919 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:43 smithi019 ceph-mon[24574]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:20:43.920 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:43 smithi019 ceph-mon[29267]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:20:44.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:43 smithi172 ceph-mon[25457]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:20:45.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:45 smithi019 ceph-mon[24574]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-02-07T12:20:45.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:45 smithi019 ceph-mon[29267]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-02-07T12:20:46.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:45 smithi172 ceph-mon[25457]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-02-07T12:20:47.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:47 smithi019 ceph-mon[29267]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-02-07T12:20:47.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:47 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-02-07T12:20:47.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:47 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:47.840 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:47 smithi019 ceph-mon[24574]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-02-07T12:20:47.840 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:47 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-02-07T12:20:47.840 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:47 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:48.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:47 smithi172 ceph-mon[25457]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-02-07T12:20:48.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:47 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-02-07T12:20:48.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:47 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:48.993 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:48 smithi019 ceph-mon[24574]: Deploying daemon osd.3 on smithi019 2024-02-07T12:20:48.993 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:48 smithi019 ceph-mon[29267]: Deploying daemon osd.3 on smithi019 2024-02-07T12:20:49.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:48 smithi172 ceph-mon[25457]: Deploying daemon osd.3 on smithi019 2024-02-07T12:20:50.093 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:49 smithi019 ceph-mon[24574]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-02-07T12:20:50.093 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:49 smithi019 ceph-mon[29267]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-02-07T12:20:50.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:49 smithi172 ceph-mon[25457]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-02-07T12:20:51.751 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:51 smithi019 ceph-mon[24574]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-02-07T12:20:51.751 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:51 smithi019 ceph-mon[29267]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-02-07T12:20:52.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:51 smithi172 ceph-mon[25457]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-02-07T12:20:53.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:52 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:53.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:52 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:20:53.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:52 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:53.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:52 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:20:53.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:52 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:53.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:52 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:20:53.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:52 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:53.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:52 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:20:53.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:52 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:53.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:52 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:20:53.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:52 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:53.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:52 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:20:54.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:53 smithi019 ceph-mon[24574]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 68 KiB/s wr, 4 op/s 2024-02-07T12:20:54.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:53 smithi019 ceph-mon[29267]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 68 KiB/s wr, 4 op/s 2024-02-07T12:20:54.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:53 smithi172 ceph-mon[25457]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 68 KiB/s wr, 4 op/s 2024-02-07T12:20:54.882 INFO:teuthology.orchestra.run.smithi019.stdout:Created osd(s) 3 on host 'smithi019' 2024-02-07T12:20:55.510 DEBUG:teuthology.orchestra.run.smithi019:osd.3> sudo journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.3.service 2024-02-07T12:20:55.513 INFO:tasks.cephadm:Deploying osd.4 on smithi172 with /dev/vg_nvme/lv_4... 2024-02-07T12:20:55.513 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-02-07T12:20:56.039 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:55 smithi019 ceph-mon[29267]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-02-07T12:20:56.195 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:55 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:56.195 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:55 smithi019 ceph-mon[29267]: from='osd.3 [v2:172.21.15.19:6826/2871829035,v1:172.21.15.19:6827/2871829035]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-02-07T12:20:56.195 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:55 smithi019 ceph-mon[29267]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-02-07T12:20:56.195 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:55 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:56.195 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:55 smithi019 ceph-mon[24574]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-02-07T12:20:56.195 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:55 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:56.195 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:55 smithi019 ceph-mon[24574]: from='osd.3 [v2:172.21.15.19:6826/2871829035,v1:172.21.15.19:6827/2871829035]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-02-07T12:20:56.195 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:55 smithi019 ceph-mon[24574]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-02-07T12:20:56.195 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:55 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:56.196 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:55 smithi172 ceph-mon[25457]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-02-07T12:20:56.196 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:55 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:56.196 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:55 smithi172 ceph-mon[25457]: from='osd.3 [v2:172.21.15.19:6826/2871829035,v1:172.21.15.19:6827/2871829035]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-02-07T12:20:56.196 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:55 smithi172 ceph-mon[25457]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-02-07T12:20:56.196 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:55 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:20:57.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:56 smithi019 ceph-mon[29267]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-02-07T12:20:57.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:56 smithi019 ceph-mon[29267]: osdmap e23: 4 total, 3 up, 4 in 2024-02-07T12:20:57.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:56 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:20:57.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:56 smithi019 ceph-mon[29267]: from='osd.3 [v2:172.21.15.19:6826/2871829035,v1:172.21.15.19:6827/2871829035]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:20:57.087 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:56 smithi019 ceph-mon[29267]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:20:57.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:56 smithi019 ceph-mon[24574]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-02-07T12:20:57.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:56 smithi019 ceph-mon[24574]: osdmap e23: 4 total, 3 up, 4 in 2024-02-07T12:20:57.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:56 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:20:57.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:56 smithi019 ceph-mon[24574]: from='osd.3 [v2:172.21.15.19:6826/2871829035,v1:172.21.15.19:6827/2871829035]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:20:57.087 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:56 smithi019 ceph-mon[24574]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:20:57.087 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:20:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3[40980]: 2024-02-07T12:20:56.893+0000 7ffa62e3d700 -1 osd.3 0 waiting for initial osdmap 2024-02-07T12:20:57.087 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:20:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3[40980]: 2024-02-07T12:20:56.897+0000 7ffa5bfd1700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:20:57.196 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:56 smithi172 ceph-mon[25457]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-02-07T12:20:57.197 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:56 smithi172 ceph-mon[25457]: osdmap e23: 4 total, 3 up, 4 in 2024-02-07T12:20:57.197 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:56 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:20:57.197 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:56 smithi172 ceph-mon[25457]: from='osd.3 [v2:172.21.15.19:6826/2871829035,v1:172.21.15.19:6827/2871829035]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:20:57.197 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:56 smithi172 ceph-mon[25457]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:20:57.492 INFO:teuthology.orchestra.run.smithi172.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-02-07T12:20:57.493 INFO:teuthology.orchestra.run.smithi172.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-02-07T12:20:57.493 INFO:teuthology.orchestra.run.smithi172.stderr: stderr: 10+0 records in 2024-02-07T12:20:57.493 INFO:teuthology.orchestra.run.smithi172.stderr:10+0 records out 2024-02-07T12:20:57.493 INFO:teuthology.orchestra.run.smithi172.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0128171 s, 818 MB/s 2024-02-07T12:20:57.493 INFO:teuthology.orchestra.run.smithi172.stderr:--> Zapping successful for: 2024-02-07T12:20:58.123 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph orch daemon add osd smithi172:vg_nvme/lv_4 2024-02-07T12:20:58.147 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:57 smithi172 ceph-mon[25457]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:20:58.148 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:57 smithi172 ceph-mon[25457]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]': finished 2024-02-07T12:20:58.148 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:57 smithi172 ceph-mon[25457]: osdmap e24: 4 total, 3 up, 4 in 2024-02-07T12:20:58.148 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:57 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:20:58.148 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:57 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:20:58.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:57 smithi019 ceph-mon[24574]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:20:58.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:57 smithi019 ceph-mon[24574]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]': finished 2024-02-07T12:20:58.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:57 smithi019 ceph-mon[24574]: osdmap e24: 4 total, 3 up, 4 in 2024-02-07T12:20:58.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:57 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:20:58.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:57 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:20:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:57 smithi019 ceph-mon[29267]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:20:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:57 smithi019 ceph-mon[29267]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]': finished 2024-02-07T12:20:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:57 smithi019 ceph-mon[29267]: osdmap e24: 4 total, 3 up, 4 in 2024-02-07T12:20:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:57 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:20:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:57 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:20:58.994 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:58 smithi019 ceph-mon[24574]: purged_snaps scrub starts 2024-02-07T12:20:58.994 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:58 smithi019 ceph-mon[24574]: purged_snaps scrub ok 2024-02-07T12:20:58.994 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:58 smithi019 ceph-mon[24574]: osd.3 [v2:172.21.15.19:6826/2871829035,v1:172.21.15.19:6827/2871829035] boot 2024-02-07T12:20:58.995 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:58 smithi019 ceph-mon[24574]: osdmap e25: 4 total, 4 up, 4 in 2024-02-07T12:20:58.995 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:58 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:20:58.995 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:58 smithi019 ceph-mon[29267]: purged_snaps scrub starts 2024-02-07T12:20:58.995 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:58 smithi019 ceph-mon[29267]: purged_snaps scrub ok 2024-02-07T12:20:58.995 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:58 smithi019 ceph-mon[29267]: osd.3 [v2:172.21.15.19:6826/2871829035,v1:172.21.15.19:6827/2871829035] boot 2024-02-07T12:20:58.995 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:58 smithi019 ceph-mon[29267]: osdmap e25: 4 total, 4 up, 4 in 2024-02-07T12:20:58.995 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:58 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:20:59.081 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:58 smithi172 ceph-mon[25457]: purged_snaps scrub starts 2024-02-07T12:20:59.081 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:58 smithi172 ceph-mon[25457]: purged_snaps scrub ok 2024-02-07T12:20:59.081 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:58 smithi172 ceph-mon[25457]: osd.3 [v2:172.21.15.19:6826/2871829035,v1:172.21.15.19:6827/2871829035] boot 2024-02-07T12:20:59.081 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:58 smithi172 ceph-mon[25457]: osdmap e25: 4 total, 4 up, 4 in 2024-02-07T12:20:59.081 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:58 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:21:00.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[24574]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:00.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[24574]: osdmap e26: 4 total, 4 up, 4 in 2024-02-07T12:21:00.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[24574]: Detected new or changed devices on smithi019 2024-02-07T12:21:00.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:00.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:21:00.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:21:00.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:21:00.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:21:00.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[24574]: Adjusting osd_memory_target on smithi019 to 4017M 2024-02-07T12:21:00.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:00.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:00.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:21:00.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:00.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:21:00.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:00.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[29267]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:00.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[29267]: osdmap e26: 4 total, 4 up, 4 in 2024-02-07T12:21:00.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[29267]: Detected new or changed devices on smithi019 2024-02-07T12:21:00.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:00.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:21:00.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:21:00.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:21:00.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:21:00.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[29267]: Adjusting osd_memory_target on smithi019 to 4017M 2024-02-07T12:21:00.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:00.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:00.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:21:00.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:00.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:21:00.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:20:59 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:00.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:59 smithi172 ceph-mon[25457]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:00.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:59 smithi172 ceph-mon[25457]: osdmap e26: 4 total, 4 up, 4 in 2024-02-07T12:21:00.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:59 smithi172 ceph-mon[25457]: Detected new or changed devices on smithi019 2024-02-07T12:21:00.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:59 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:00.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:59 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:21:00.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:59 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:21:00.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:59 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:21:00.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:59 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:21:00.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:59 smithi172 ceph-mon[25457]: Adjusting osd_memory_target on smithi019 to 4017M 2024-02-07T12:21:00.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:59 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:00.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:59 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:00.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:59 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:21:00.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:59 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:00.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:59 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:21:00.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:20:59 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:01.056 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:00 smithi172 ceph-mon[25457]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi172:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:21:01.057 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:00 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:21:01.057 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:00 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:21:01.057 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:00 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:01.057 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:00 smithi172 ceph-mon[25457]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:01.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:00 smithi019 ceph-mon[24574]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi172:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:21:01.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:00 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:21:01.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:00 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:21:01.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:00 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:01.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:00 smithi019 ceph-mon[24574]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:01.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:00 smithi019 ceph-mon[29267]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi172:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:21:01.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:00 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:21:01.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:00 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:21:01.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:00 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:01.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:00 smithi019 ceph-mon[29267]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:02.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:02 smithi019 ceph-mon[24574]: from='client.? 172.21.15.172:0/490652849' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7b6de577-7c35-4cc0-bf7e-cddbffbaad2b"}]: dispatch 2024-02-07T12:21:02.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:02 smithi019 ceph-mon[24574]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7b6de577-7c35-4cc0-bf7e-cddbffbaad2b"}]: dispatch 2024-02-07T12:21:02.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:02 smithi019 ceph-mon[24574]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7b6de577-7c35-4cc0-bf7e-cddbffbaad2b"}]': finished 2024-02-07T12:21:02.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:02 smithi019 ceph-mon[24574]: osdmap e27: 5 total, 4 up, 5 in 2024-02-07T12:21:02.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:02 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:21:02.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:02 smithi019 ceph-mon[29267]: from='client.? 172.21.15.172:0/490652849' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7b6de577-7c35-4cc0-bf7e-cddbffbaad2b"}]: dispatch 2024-02-07T12:21:02.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:02 smithi019 ceph-mon[29267]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7b6de577-7c35-4cc0-bf7e-cddbffbaad2b"}]: dispatch 2024-02-07T12:21:02.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:02 smithi019 ceph-mon[29267]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7b6de577-7c35-4cc0-bf7e-cddbffbaad2b"}]': finished 2024-02-07T12:21:02.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:02 smithi019 ceph-mon[29267]: osdmap e27: 5 total, 4 up, 5 in 2024-02-07T12:21:02.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:02 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:21:02.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:02 smithi172 ceph-mon[25457]: from='client.? 172.21.15.172:0/490652849' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7b6de577-7c35-4cc0-bf7e-cddbffbaad2b"}]: dispatch 2024-02-07T12:21:02.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:02 smithi172 ceph-mon[25457]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7b6de577-7c35-4cc0-bf7e-cddbffbaad2b"}]: dispatch 2024-02-07T12:21:02.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:02 smithi172 ceph-mon[25457]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7b6de577-7c35-4cc0-bf7e-cddbffbaad2b"}]': finished 2024-02-07T12:21:02.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:02 smithi172 ceph-mon[25457]: osdmap e27: 5 total, 4 up, 5 in 2024-02-07T12:21:02.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:02 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:21:04.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:03 smithi019 ceph-mon[29267]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:04.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:03 smithi019 ceph-mon[29267]: from='client.? 172.21.15.172:0/2047685734' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:21:04.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:03 smithi019 ceph-mon[24574]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:04.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:03 smithi019 ceph-mon[24574]: from='client.? 172.21.15.172:0/2047685734' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:21:04.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:03 smithi172 ceph-mon[25457]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:04.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:03 smithi172 ceph-mon[25457]: from='client.? 172.21.15.172:0/2047685734' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:21:05.731 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:05 smithi172 ceph-mon[25457]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:06.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:05 smithi019 ceph-mon[24574]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:06.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:05 smithi019 ceph-mon[29267]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:07.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:07 smithi172 ceph-mon[25457]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:08.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:07 smithi019 ceph-mon[24574]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:08.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:07 smithi019 ceph-mon[29267]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:08.905 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:08 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-02-07T12:21:08.905 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:08 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:09.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:08 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-02-07T12:21:09.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:08 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:09.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:08 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-02-07T12:21:09.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:08 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:09.902 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:09 smithi172 ceph-mon[25457]: Deploying daemon osd.4 on smithi172 2024-02-07T12:21:09.903 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:09 smithi172 ceph-mon[25457]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:10.073 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:09 smithi019 ceph-mon[29267]: Deploying daemon osd.4 on smithi172 2024-02-07T12:21:10.073 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:09 smithi019 ceph-mon[29267]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:10.073 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:09 smithi019 ceph-mon[24574]: Deploying daemon osd.4 on smithi172 2024-02-07T12:21:10.073 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:09 smithi019 ceph-mon[24574]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:12.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:11 smithi019 ceph-mon[24574]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:12.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:11 smithi019 ceph-mon[29267]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:12.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:11 smithi172 ceph-mon[25457]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:14.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:13 smithi172 ceph-mon[25457]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:14.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:13 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:14.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:13 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:21:14.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:13 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:14.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:13 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:21:14.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:13 smithi019 ceph-mon[24574]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:14.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:13 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:14.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:13 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:21:14.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:13 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:14.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:13 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:21:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:13 smithi019 ceph-mon[29267]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:13 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:13 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:21:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:13 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:13 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:21:16.090 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:15 smithi172 ceph-mon[25457]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:16.090 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:15 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:16.090 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:15 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:16.119 INFO:teuthology.orchestra.run.smithi172.stdout:Created osd(s) 4 on host 'smithi172' 2024-02-07T12:21:16.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:15 smithi019 ceph-mon[24574]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:16.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:15 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:16.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:15 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:16.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:15 smithi019 ceph-mon[29267]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:16.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:15 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:16.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:15 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:16.749 DEBUG:teuthology.orchestra.run.smithi172:osd.4> sudo journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.4.service 2024-02-07T12:21:16.791 INFO:tasks.cephadm:Deploying osd.5 on smithi172 with /dev/vg_nvme/lv_3... 2024-02-07T12:21:16.791 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-02-07T12:21:17.093 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:16 smithi172 ceph-mon[25457]: from='osd.4 [v2:172.21.15.172:6800/2804555213,v1:172.21.15.172:6801/2804555213]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-02-07T12:21:17.093 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:16 smithi172 ceph-mon[25457]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-02-07T12:21:17.093 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:16 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:17.093 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:16 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:21:17.093 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:16 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:17.094 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:16 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:21:17.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:16 smithi019 ceph-mon[29267]: from='osd.4 [v2:172.21.15.172:6800/2804555213,v1:172.21.15.172:6801/2804555213]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-02-07T12:21:17.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:16 smithi019 ceph-mon[29267]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-02-07T12:21:17.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:16 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:17.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:16 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:21:17.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:16 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:17.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:16 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:21:17.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:16 smithi019 ceph-mon[24574]: from='osd.4 [v2:172.21.15.172:6800/2804555213,v1:172.21.15.172:6801/2804555213]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-02-07T12:21:17.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:16 smithi019 ceph-mon[24574]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-02-07T12:21:17.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:16 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:17.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:16 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:21:17.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:16 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:17.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:16 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:21:18.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:17 smithi019 ceph-mon[29267]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:18.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:17 smithi019 ceph-mon[29267]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-02-07T12:21:18.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:17 smithi019 ceph-mon[29267]: osdmap e28: 5 total, 4 up, 5 in 2024-02-07T12:21:18.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:17 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:21:18.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:17 smithi019 ceph-mon[29267]: from='osd.4 [v2:172.21.15.172:6800/2804555213,v1:172.21.15.172:6801/2804555213]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:21:18.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:17 smithi019 ceph-mon[29267]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:21:18.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:17 smithi019 ceph-mon[24574]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:18.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:17 smithi019 ceph-mon[24574]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-02-07T12:21:18.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:17 smithi019 ceph-mon[24574]: osdmap e28: 5 total, 4 up, 5 in 2024-02-07T12:21:18.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:17 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:21:18.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:17 smithi019 ceph-mon[24574]: from='osd.4 [v2:172.21.15.172:6800/2804555213,v1:172.21.15.172:6801/2804555213]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:21:18.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:17 smithi019 ceph-mon[24574]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:21:18.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:17 smithi172 ceph-mon[25457]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:18.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:17 smithi172 ceph-mon[25457]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-02-07T12:21:18.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:17 smithi172 ceph-mon[25457]: osdmap e28: 5 total, 4 up, 5 in 2024-02-07T12:21:18.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:17 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:21:18.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:17 smithi172 ceph-mon[25457]: from='osd.4 [v2:172.21.15.172:6800/2804555213,v1:172.21.15.172:6801/2804555213]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:21:18.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:17 smithi172 ceph-mon[25457]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:21:18.370 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:21:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4[28829]: 2024-02-07T12:21:17.880+0000 7fb3e56db700 -1 osd.4 0 waiting for initial osdmap 2024-02-07T12:21:18.370 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:21:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4[28829]: 2024-02-07T12:21:17.884+0000 7fb3e0873700 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:21:18.923 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:18 smithi172 ceph-mon[25457]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]': finished 2024-02-07T12:21:18.923 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:18 smithi172 ceph-mon[25457]: osdmap e29: 5 total, 4 up, 5 in 2024-02-07T12:21:18.923 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:18 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:21:18.923 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:18 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:21:19.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:18 smithi019 ceph-mon[24574]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]': finished 2024-02-07T12:21:19.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:18 smithi019 ceph-mon[24574]: osdmap e29: 5 total, 4 up, 5 in 2024-02-07T12:21:19.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:18 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:21:19.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:18 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:21:19.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:18 smithi019 ceph-mon[29267]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]': finished 2024-02-07T12:21:19.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:18 smithi019 ceph-mon[29267]: osdmap e29: 5 total, 4 up, 5 in 2024-02-07T12:21:19.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:18 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:21:19.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:18 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:21:20.056 INFO:teuthology.orchestra.run.smithi172.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-02-07T12:21:20.056 INFO:teuthology.orchestra.run.smithi172.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-02-07T12:21:20.056 INFO:teuthology.orchestra.run.smithi172.stderr: stderr: 10+0 records in 2024-02-07T12:21:20.056 INFO:teuthology.orchestra.run.smithi172.stderr:10+0 records out 2024-02-07T12:21:20.056 INFO:teuthology.orchestra.run.smithi172.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0159038 s, 659 MB/s 2024-02-07T12:21:20.056 INFO:teuthology.orchestra.run.smithi172.stderr:--> Zapping successful for: 2024-02-07T12:21:20.157 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:19 smithi172 ceph-mon[25457]: purged_snaps scrub starts 2024-02-07T12:21:20.157 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:19 smithi172 ceph-mon[25457]: purged_snaps scrub ok 2024-02-07T12:21:20.157 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:19 smithi172 ceph-mon[25457]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:20.157 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:19 smithi172 ceph-mon[25457]: osd.4 [v2:172.21.15.172:6800/2804555213,v1:172.21.15.172:6801/2804555213] boot 2024-02-07T12:21:20.157 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:19 smithi172 ceph-mon[25457]: osdmap e30: 5 total, 5 up, 5 in 2024-02-07T12:21:20.157 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:19 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:21:20.157 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:19 smithi172 ceph-mon[25457]: osdmap e31: 5 total, 5 up, 5 in 2024-02-07T12:21:20.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:19 smithi019 ceph-mon[24574]: purged_snaps scrub starts 2024-02-07T12:21:20.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:19 smithi019 ceph-mon[24574]: purged_snaps scrub ok 2024-02-07T12:21:20.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:19 smithi019 ceph-mon[24574]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:20.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:19 smithi019 ceph-mon[24574]: osd.4 [v2:172.21.15.172:6800/2804555213,v1:172.21.15.172:6801/2804555213] boot 2024-02-07T12:21:20.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:19 smithi019 ceph-mon[24574]: osdmap e30: 5 total, 5 up, 5 in 2024-02-07T12:21:20.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:19 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:21:20.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:19 smithi019 ceph-mon[24574]: osdmap e31: 5 total, 5 up, 5 in 2024-02-07T12:21:20.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:19 smithi019 ceph-mon[29267]: purged_snaps scrub starts 2024-02-07T12:21:20.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:19 smithi019 ceph-mon[29267]: purged_snaps scrub ok 2024-02-07T12:21:20.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:19 smithi019 ceph-mon[29267]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:21:20.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:19 smithi019 ceph-mon[29267]: osd.4 [v2:172.21.15.172:6800/2804555213,v1:172.21.15.172:6801/2804555213] boot 2024-02-07T12:21:20.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:19 smithi019 ceph-mon[29267]: osdmap e30: 5 total, 5 up, 5 in 2024-02-07T12:21:20.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:19 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:21:20.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:19 smithi019 ceph-mon[29267]: osdmap e31: 5 total, 5 up, 5 in 2024-02-07T12:21:21.279 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph orch daemon add osd smithi172:vg_nvme/lv_3 2024-02-07T12:21:21.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:21 smithi172 ceph-mon[25457]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:21:21.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:21 smithi172 ceph-mon[25457]: Detected new or changed devices on smithi172 2024-02-07T12:21:21.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:21 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:21.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:21 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:21:21.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:21 smithi172 ceph-mon[25457]: Adjusting osd_memory_target on smithi172 to 17088M 2024-02-07T12:21:21.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:21 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:21.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:21 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:21.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:21 smithi172 ceph-mon[25457]: osdmap e32: 5 total, 5 up, 5 in 2024-02-07T12:21:22.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:21 smithi019 ceph-mon[24574]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:21:22.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:21 smithi019 ceph-mon[24574]: Detected new or changed devices on smithi172 2024-02-07T12:21:22.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:21 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:22.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:21 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:21:22.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:21 smithi019 ceph-mon[24574]: Adjusting osd_memory_target on smithi172 to 17088M 2024-02-07T12:21:22.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:21 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:22.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:21 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:22.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:21 smithi019 ceph-mon[24574]: osdmap e32: 5 total, 5 up, 5 in 2024-02-07T12:21:22.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:21 smithi019 ceph-mon[29267]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:21:22.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:21 smithi019 ceph-mon[29267]: Detected new or changed devices on smithi172 2024-02-07T12:21:22.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:21 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:22.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:21 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:21:22.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:21 smithi019 ceph-mon[29267]: Adjusting osd_memory_target on smithi172 to 17088M 2024-02-07T12:21:22.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:21 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:22.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:21 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:22.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:21 smithi019 ceph-mon[29267]: osdmap e32: 5 total, 5 up, 5 in 2024-02-07T12:21:23.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:22 smithi172 ceph-mon[25457]: osdmap e33: 5 total, 5 up, 5 in 2024-02-07T12:21:23.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:22 smithi019 ceph-mon[24574]: osdmap e33: 5 total, 5 up, 5 in 2024-02-07T12:21:23.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:22 smithi019 ceph-mon[29267]: osdmap e33: 5 total, 5 up, 5 in 2024-02-07T12:21:24.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:23 smithi019 ceph-mon[24574]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:21:24.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:23 smithi019 ceph-mon[24574]: osdmap e34: 5 total, 5 up, 5 in 2024-02-07T12:21:24.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:23 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:21:24.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:23 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:21:24.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:23 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:24.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:23 smithi019 ceph-mon[29267]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:21:24.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:23 smithi019 ceph-mon[29267]: osdmap e34: 5 total, 5 up, 5 in 2024-02-07T12:21:24.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:23 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:21:24.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:23 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:21:24.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:23 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:24.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:23 smithi172 ceph-mon[25457]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:21:24.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:23 smithi172 ceph-mon[25457]: osdmap e34: 5 total, 5 up, 5 in 2024-02-07T12:21:24.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:23 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:21:24.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:23 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:21:24.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:23 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:25.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:24 smithi019 ceph-mon[24574]: from='client.24230 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi172:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:21:25.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:24 smithi019 ceph-mon[29267]: from='client.24230 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi172:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:21:25.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:24 smithi172 ceph-mon[25457]: from='client.24230 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi172:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:21:26.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:25 smithi019 ceph-mon[24574]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 108 KiB/s, 0 objects/s recovering 2024-02-07T12:21:26.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:25 smithi019 ceph-mon[24574]: from='client.? 172.21.15.172:0/869049805' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "efa49cf9-057d-4a6c-b3b8-368b9497f6ff"}]: dispatch 2024-02-07T12:21:26.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:25 smithi019 ceph-mon[24574]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "efa49cf9-057d-4a6c-b3b8-368b9497f6ff"}]: dispatch 2024-02-07T12:21:26.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:25 smithi019 ceph-mon[24574]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "efa49cf9-057d-4a6c-b3b8-368b9497f6ff"}]': finished 2024-02-07T12:21:26.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:25 smithi019 ceph-mon[24574]: osdmap e35: 6 total, 5 up, 6 in 2024-02-07T12:21:26.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:25 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:21:26.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:25 smithi019 ceph-mon[29267]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 108 KiB/s, 0 objects/s recovering 2024-02-07T12:21:26.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:25 smithi019 ceph-mon[29267]: from='client.? 172.21.15.172:0/869049805' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "efa49cf9-057d-4a6c-b3b8-368b9497f6ff"}]: dispatch 2024-02-07T12:21:26.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:25 smithi019 ceph-mon[29267]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "efa49cf9-057d-4a6c-b3b8-368b9497f6ff"}]: dispatch 2024-02-07T12:21:26.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:25 smithi019 ceph-mon[29267]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "efa49cf9-057d-4a6c-b3b8-368b9497f6ff"}]': finished 2024-02-07T12:21:26.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:25 smithi019 ceph-mon[29267]: osdmap e35: 6 total, 5 up, 6 in 2024-02-07T12:21:26.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:25 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:21:26.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:25 smithi172 ceph-mon[25457]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 108 KiB/s, 0 objects/s recovering 2024-02-07T12:21:26.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:25 smithi172 ceph-mon[25457]: from='client.? 172.21.15.172:0/869049805' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "efa49cf9-057d-4a6c-b3b8-368b9497f6ff"}]: dispatch 2024-02-07T12:21:26.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:25 smithi172 ceph-mon[25457]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "efa49cf9-057d-4a6c-b3b8-368b9497f6ff"}]: dispatch 2024-02-07T12:21:26.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:25 smithi172 ceph-mon[25457]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "efa49cf9-057d-4a6c-b3b8-368b9497f6ff"}]': finished 2024-02-07T12:21:26.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:25 smithi172 ceph-mon[25457]: osdmap e35: 6 total, 5 up, 6 in 2024-02-07T12:21:26.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:25 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:21:27.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:26 smithi019 ceph-mon[24574]: from='client.? 172.21.15.172:0/1857829191' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:21:27.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:26 smithi019 ceph-mon[24574]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-02-07T12:21:27.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:26 smithi019 ceph-mon[29267]: from='client.? 172.21.15.172:0/1857829191' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:21:27.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:26 smithi019 ceph-mon[29267]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-02-07T12:21:27.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:26 smithi172 ceph-mon[25457]: from='client.? 172.21.15.172:0/1857829191' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:21:27.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:26 smithi172 ceph-mon[25457]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-02-07T12:21:29.965 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:29 smithi172 ceph-mon[25457]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-02-07T12:21:29.965 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:29 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:21:29.965 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:29 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:21:30.048 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:29 smithi019 ceph-mon[29267]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-02-07T12:21:30.048 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:29 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:21:30.048 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:29 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:21:30.048 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:29 smithi019 ceph-mon[24574]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-02-07T12:21:30.049 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:29 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:21:30.049 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:29 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:21:31.860 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:31 smithi172 ceph-mon[25457]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-02-07T12:21:31.860 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:31 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-02-07T12:21:31.860 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:31 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:32.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:31 smithi019 ceph-mon[24574]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-02-07T12:21:32.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:31 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-02-07T12:21:32.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:31 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:32.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:31 smithi019 ceph-mon[29267]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-02-07T12:21:32.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:31 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-02-07T12:21:32.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:31 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:33.002 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:32 smithi172 ceph-mon[25457]: Deploying daemon osd.5 on smithi172 2024-02-07T12:21:33.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:32 smithi019 ceph-mon[24574]: Deploying daemon osd.5 on smithi172 2024-02-07T12:21:33.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:32 smithi019 ceph-mon[29267]: Deploying daemon osd.5 on smithi172 2024-02-07T12:21:33.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:33 smithi172 ceph-mon[25457]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:21:34.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:33 smithi019 ceph-mon[24574]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:21:34.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:33 smithi019 ceph-mon[29267]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:21:35.799 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:35 smithi172 ceph-mon[25457]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:21:36.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:35 smithi019 ceph-mon[24574]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:21:36.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:35 smithi019 ceph-mon[29267]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:21:38.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:37 smithi019 ceph-mon[24574]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:21:38.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:37 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:38.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:37 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:21:38.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:37 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:38.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:37 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:21:38.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:37 smithi019 ceph-mon[29267]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:21:38.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:37 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:38.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:37 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:21:38.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:37 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:38.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:37 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:21:38.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:37 smithi172 ceph-mon[25457]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:21:38.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:37 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:38.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:37 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:21:38.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:37 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:38.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:37 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:21:40.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:39 smithi019 ceph-mon[24574]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:21:40.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:39 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:40.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:39 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:40.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:39 smithi019 ceph-mon[29267]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:21:40.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:39 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:40.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:39 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:40.101 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:39 smithi172 ceph-mon[25457]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:21:40.101 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:39 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:40.101 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:39 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:40.132 INFO:teuthology.orchestra.run.smithi172.stdout:Created osd(s) 5 on host 'smithi172' 2024-02-07T12:21:40.786 DEBUG:teuthology.orchestra.run.smithi172:osd.5> sudo journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.5.service 2024-02-07T12:21:40.788 INFO:tasks.cephadm:Deploying osd.6 on smithi172 with /dev/vg_nvme/lv_2... 2024-02-07T12:21:40.788 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-02-07T12:21:40.989 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:40 smithi172 ceph-mon[25457]: from='osd.5 [v2:172.21.15.172:6808/3717679655,v1:172.21.15.172:6809/3717679655]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-07T12:21:40.989 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:40 smithi172 ceph-mon[25457]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-07T12:21:40.989 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:40 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:40.989 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:40 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:21:40.989 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:40 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:40.989 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:40 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:21:41.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:40 smithi019 ceph-mon[24574]: from='osd.5 [v2:172.21.15.172:6808/3717679655,v1:172.21.15.172:6809/3717679655]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-07T12:21:41.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:40 smithi019 ceph-mon[24574]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-07T12:21:41.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:40 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:41.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:40 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:21:41.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:40 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:41.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:40 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:21:41.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:40 smithi019 ceph-mon[29267]: from='osd.5 [v2:172.21.15.172:6808/3717679655,v1:172.21.15.172:6809/3717679655]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-07T12:21:41.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:40 smithi019 ceph-mon[29267]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-07T12:21:41.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:40 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:41.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:40 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:21:41.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:40 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:41.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:40 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:21:41.845 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:41 smithi172 ceph-mon[25457]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:21:41.845 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:41 smithi172 ceph-mon[25457]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-02-07T12:21:41.845 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:41 smithi172 ceph-mon[25457]: osdmap e36: 6 total, 5 up, 6 in 2024-02-07T12:21:41.845 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:41 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:21:41.845 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:41 smithi172 ceph-mon[25457]: from='osd.5 [v2:172.21.15.172:6808/3717679655,v1:172.21.15.172:6809/3717679655]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:21:41.845 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:41 smithi172 ceph-mon[25457]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:21:41.846 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:21:41 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[31771]: 2024-02-07T12:21:41.730+0000 7f02cd8f0700 -1 osd.5 0 waiting for initial osdmap 2024-02-07T12:21:41.846 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:21:41 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[31771]: 2024-02-07T12:21:41.734+0000 7f02c8a88700 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:21:42.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:41 smithi019 ceph-mon[24574]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:21:42.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:41 smithi019 ceph-mon[24574]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-02-07T12:21:42.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:41 smithi019 ceph-mon[24574]: osdmap e36: 6 total, 5 up, 6 in 2024-02-07T12:21:42.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:41 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:21:42.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:41 smithi019 ceph-mon[24574]: from='osd.5 [v2:172.21.15.172:6808/3717679655,v1:172.21.15.172:6809/3717679655]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:21:42.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:41 smithi019 ceph-mon[24574]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:21:42.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:41 smithi019 ceph-mon[29267]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:21:42.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:41 smithi019 ceph-mon[29267]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-02-07T12:21:42.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:41 smithi019 ceph-mon[29267]: osdmap e36: 6 total, 5 up, 6 in 2024-02-07T12:21:42.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:41 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:21:42.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:41 smithi019 ceph-mon[29267]: from='osd.5 [v2:172.21.15.172:6808/3717679655,v1:172.21.15.172:6809/3717679655]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:21:42.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:41 smithi019 ceph-mon[29267]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:21:43.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:42 smithi019 ceph-mon[24574]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]': finished 2024-02-07T12:21:43.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:42 smithi019 ceph-mon[24574]: osdmap e37: 6 total, 5 up, 6 in 2024-02-07T12:21:43.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:42 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:21:43.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:42 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:21:43.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:42 smithi019 ceph-mon[24574]: osd.5 [v2:172.21.15.172:6808/3717679655,v1:172.21.15.172:6809/3717679655] boot 2024-02-07T12:21:43.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:42 smithi019 ceph-mon[24574]: osdmap e38: 6 total, 6 up, 6 in 2024-02-07T12:21:43.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:42 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:21:43.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:42 smithi019 ceph-mon[29267]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]': finished 2024-02-07T12:21:43.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:42 smithi019 ceph-mon[29267]: osdmap e37: 6 total, 5 up, 6 in 2024-02-07T12:21:43.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:42 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:21:43.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:42 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:21:43.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:42 smithi019 ceph-mon[29267]: osd.5 [v2:172.21.15.172:6808/3717679655,v1:172.21.15.172:6809/3717679655] boot 2024-02-07T12:21:43.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:42 smithi019 ceph-mon[29267]: osdmap e38: 6 total, 6 up, 6 in 2024-02-07T12:21:43.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:42 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:21:43.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:42 smithi172 ceph-mon[25457]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]': finished 2024-02-07T12:21:43.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:42 smithi172 ceph-mon[25457]: osdmap e37: 6 total, 5 up, 6 in 2024-02-07T12:21:43.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:42 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:21:43.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:42 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:21:43.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:42 smithi172 ceph-mon[25457]: osd.5 [v2:172.21.15.172:6808/3717679655,v1:172.21.15.172:6809/3717679655] boot 2024-02-07T12:21:43.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:42 smithi172 ceph-mon[25457]: osdmap e38: 6 total, 6 up, 6 in 2024-02-07T12:21:43.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:42 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:21:43.224 INFO:teuthology.orchestra.run.smithi172.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-02-07T12:21:43.224 INFO:teuthology.orchestra.run.smithi172.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-02-07T12:21:43.224 INFO:teuthology.orchestra.run.smithi172.stderr: stderr: 10+0 records in 2024-02-07T12:21:43.224 INFO:teuthology.orchestra.run.smithi172.stderr:10+0 records out 2024-02-07T12:21:43.224 INFO:teuthology.orchestra.run.smithi172.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0122536 s, 856 MB/s 2024-02-07T12:21:43.224 INFO:teuthology.orchestra.run.smithi172.stderr:--> Zapping successful for: 2024-02-07T12:21:44.050 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:43 smithi172 ceph-mon[25457]: purged_snaps scrub starts 2024-02-07T12:21:44.050 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:43 smithi172 ceph-mon[25457]: purged_snaps scrub ok 2024-02-07T12:21:44.050 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:43 smithi172 ceph-mon[25457]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:21:44.050 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:43 smithi172 ceph-mon[25457]: osdmap e39: 6 total, 6 up, 6 in 2024-02-07T12:21:44.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:43 smithi019 ceph-mon[24574]: purged_snaps scrub starts 2024-02-07T12:21:44.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:43 smithi019 ceph-mon[24574]: purged_snaps scrub ok 2024-02-07T12:21:44.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:43 smithi019 ceph-mon[24574]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:21:44.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:43 smithi019 ceph-mon[24574]: osdmap e39: 6 total, 6 up, 6 in 2024-02-07T12:21:44.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:43 smithi019 ceph-mon[29267]: purged_snaps scrub starts 2024-02-07T12:21:44.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:43 smithi019 ceph-mon[29267]: purged_snaps scrub ok 2024-02-07T12:21:44.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:43 smithi019 ceph-mon[29267]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:21:44.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:43 smithi019 ceph-mon[29267]: osdmap e39: 6 total, 6 up, 6 in 2024-02-07T12:21:44.469 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph orch daemon add osd smithi172:vg_nvme/lv_2 2024-02-07T12:21:45.810 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:45 smithi172 ceph-mon[25457]: pgmap v127: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:21:45.810 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:45 smithi172 ceph-mon[25457]: osdmap e40: 6 total, 6 up, 6 in 2024-02-07T12:21:45.810 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:45 smithi172 ceph-mon[25457]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-02-07T12:21:46.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:45 smithi019 ceph-mon[24574]: pgmap v127: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:21:46.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:45 smithi019 ceph-mon[24574]: osdmap e40: 6 total, 6 up, 6 in 2024-02-07T12:21:46.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:45 smithi019 ceph-mon[24574]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-02-07T12:21:46.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:45 smithi019 ceph-mon[29267]: pgmap v127: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:21:46.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:45 smithi019 ceph-mon[29267]: osdmap e40: 6 total, 6 up, 6 in 2024-02-07T12:21:46.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:45 smithi019 ceph-mon[29267]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-02-07T12:21:47.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:46 smithi019 ceph-mon[24574]: osdmap e41: 6 total, 6 up, 6 in 2024-02-07T12:21:47.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:46 smithi019 ceph-mon[24574]: Detected new or changed devices on smithi172 2024-02-07T12:21:47.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:46 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:47.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:46 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:21:47.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:46 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:21:47.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:46 smithi019 ceph-mon[24574]: Adjusting osd_memory_target on smithi172 to 8544M 2024-02-07T12:21:47.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:46 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:47.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:46 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:47.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:46 smithi019 ceph-mon[29267]: osdmap e41: 6 total, 6 up, 6 in 2024-02-07T12:21:47.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:46 smithi019 ceph-mon[29267]: Detected new or changed devices on smithi172 2024-02-07T12:21:47.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:46 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:47.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:46 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:21:47.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:46 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:21:47.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:46 smithi019 ceph-mon[29267]: Adjusting osd_memory_target on smithi172 to 8544M 2024-02-07T12:21:47.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:46 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:47.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:46 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:46 smithi172 ceph-mon[25457]: osdmap e41: 6 total, 6 up, 6 in 2024-02-07T12:21:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:46 smithi172 ceph-mon[25457]: Detected new or changed devices on smithi172 2024-02-07T12:21:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:46 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:46 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:21:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:46 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:21:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:46 smithi172 ceph-mon[25457]: Adjusting osd_memory_target on smithi172 to 8544M 2024-02-07T12:21:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:46 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:47.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:46 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:21:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:47 smithi172 ceph-mon[25457]: pgmap v130: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:21:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:47 smithi172 ceph-mon[25457]: osdmap e42: 6 total, 6 up, 6 in 2024-02-07T12:21:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:47 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:21:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:47 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:21:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:47 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:48.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:47 smithi019 ceph-mon[24574]: pgmap v130: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:21:48.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:47 smithi019 ceph-mon[24574]: osdmap e42: 6 total, 6 up, 6 in 2024-02-07T12:21:48.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:47 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:21:48.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:47 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:21:48.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:47 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:48.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:47 smithi019 ceph-mon[29267]: pgmap v130: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:21:48.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:47 smithi019 ceph-mon[29267]: osdmap e42: 6 total, 6 up, 6 in 2024-02-07T12:21:48.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:47 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:21:48.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:47 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:21:48.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:47 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:48.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:48 smithi172 ceph-mon[25457]: from='client.24257 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi172:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:21:49.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:48 smithi019 ceph-mon[24574]: from='client.24257 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi172:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:21:49.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:48 smithi019 ceph-mon[29267]: from='client.24257 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi172:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:21:50.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:49 smithi019 ceph-mon[24574]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:21:50.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:49 smithi019 ceph-mon[24574]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-02-07T12:21:50.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:49 smithi019 ceph-mon[24574]: Cluster is now healthy 2024-02-07T12:21:50.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:49 smithi019 ceph-mon[24574]: from='client.? 172.21.15.172:0/2567308352' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fe43b6a2-9866-4da0-a167-f4ba5564cdbc"}]: dispatch 2024-02-07T12:21:50.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:49 smithi019 ceph-mon[24574]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fe43b6a2-9866-4da0-a167-f4ba5564cdbc"}]: dispatch 2024-02-07T12:21:50.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:49 smithi019 ceph-mon[24574]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fe43b6a2-9866-4da0-a167-f4ba5564cdbc"}]': finished 2024-02-07T12:21:50.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:49 smithi019 ceph-mon[24574]: osdmap e43: 7 total, 6 up, 7 in 2024-02-07T12:21:50.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:49 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:21:50.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:49 smithi019 ceph-mon[29267]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:21:50.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:49 smithi019 ceph-mon[29267]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-02-07T12:21:50.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:49 smithi019 ceph-mon[29267]: Cluster is now healthy 2024-02-07T12:21:50.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:49 smithi019 ceph-mon[29267]: from='client.? 172.21.15.172:0/2567308352' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fe43b6a2-9866-4da0-a167-f4ba5564cdbc"}]: dispatch 2024-02-07T12:21:50.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:49 smithi019 ceph-mon[29267]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fe43b6a2-9866-4da0-a167-f4ba5564cdbc"}]: dispatch 2024-02-07T12:21:50.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:49 smithi019 ceph-mon[29267]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fe43b6a2-9866-4da0-a167-f4ba5564cdbc"}]': finished 2024-02-07T12:21:50.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:49 smithi019 ceph-mon[29267]: osdmap e43: 7 total, 6 up, 7 in 2024-02-07T12:21:50.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:49 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:21:50.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:49 smithi172 ceph-mon[25457]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:21:50.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:49 smithi172 ceph-mon[25457]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-02-07T12:21:50.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:49 smithi172 ceph-mon[25457]: Cluster is now healthy 2024-02-07T12:21:50.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:49 smithi172 ceph-mon[25457]: from='client.? 172.21.15.172:0/2567308352' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fe43b6a2-9866-4da0-a167-f4ba5564cdbc"}]: dispatch 2024-02-07T12:21:50.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:49 smithi172 ceph-mon[25457]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fe43b6a2-9866-4da0-a167-f4ba5564cdbc"}]: dispatch 2024-02-07T12:21:50.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:49 smithi172 ceph-mon[25457]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fe43b6a2-9866-4da0-a167-f4ba5564cdbc"}]': finished 2024-02-07T12:21:50.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:49 smithi172 ceph-mon[25457]: osdmap e43: 7 total, 6 up, 7 in 2024-02-07T12:21:50.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:49 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:21:51.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:50 smithi019 ceph-mon[24574]: from='client.? 172.21.15.172:0/731836532' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:21:51.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:50 smithi019 ceph-mon[29267]: from='client.? 172.21.15.172:0/731836532' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:21:51.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:50 smithi172 ceph-mon[25457]: from='client.? 172.21.15.172:0/731836532' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:21:52.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:51 smithi019 ceph-mon[24574]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-02-07T12:21:52.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:51 smithi019 ceph-mon[29267]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-02-07T12:21:52.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:51 smithi172 ceph-mon[25457]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-02-07T12:21:53.845 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:53 smithi172 ceph-mon[25457]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-02-07T12:21:54.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:53 smithi019 ceph-mon[24574]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-02-07T12:21:54.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:53 smithi019 ceph-mon[29267]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-02-07T12:21:55.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:55 smithi172 ceph-mon[25457]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-02-07T12:21:55.872 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:55 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-02-07T12:21:55.872 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:55 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:56.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:55 smithi019 ceph-mon[24574]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-02-07T12:21:56.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:55 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-02-07T12:21:56.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:55 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:56.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:55 smithi019 ceph-mon[29267]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-02-07T12:21:56.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:55 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-02-07T12:21:56.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:55 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:56.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:56 smithi172 ceph-mon[25457]: Deploying daemon osd.6 on smithi172 2024-02-07T12:21:57.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:56 smithi019 ceph-mon[24574]: Deploying daemon osd.6 on smithi172 2024-02-07T12:21:57.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:56 smithi019 ceph-mon[29267]: Deploying daemon osd.6 on smithi172 2024-02-07T12:21:57.994 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:57 smithi172 ceph-mon[25457]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-02-07T12:21:58.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:57 smithi019 ceph-mon[24574]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-02-07T12:21:58.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:57 smithi019 ceph-mon[29267]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-02-07T12:21:59.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:21:59 smithi172 ceph-mon[25457]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:22:00.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:21:59 smithi019 ceph-mon[24574]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:22:00.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:21:59 smithi019 ceph-mon[29267]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:22:01.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:01 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:01.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:01 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:22:01.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:01 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:01.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:01 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:22:01.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:01 smithi019 ceph-mon[29267]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:22:01.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:01 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:01.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:01 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:22:01.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:01 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:01.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:01 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:22:01.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:01 smithi019 ceph-mon[24574]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:22:01.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:01 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:01.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:01 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:22:01.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:01 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:01.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:01 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:22:01.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:01 smithi172 ceph-mon[25457]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:22:03.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:03 smithi172 ceph-mon[25457]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:22:03.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:03 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:03.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:03 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:03.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:03 smithi172 ceph-mon[25457]: from='osd.6 [v2:172.21.15.172:6816/92131068,v1:172.21.15.172:6817/92131068]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-02-07T12:22:03.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:03 smithi172 ceph-mon[25457]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-02-07T12:22:03.968 INFO:teuthology.orchestra.run.smithi172.stdout:Created osd(s) 6 on host 'smithi172' 2024-02-07T12:22:04.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:03 smithi019 ceph-mon[29267]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:22:04.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:03 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:04.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:03 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:04.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:03 smithi019 ceph-mon[29267]: from='osd.6 [v2:172.21.15.172:6816/92131068,v1:172.21.15.172:6817/92131068]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-02-07T12:22:04.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:03 smithi019 ceph-mon[29267]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-02-07T12:22:04.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:03 smithi019 ceph-mon[24574]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:22:04.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:03 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:04.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:03 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:04.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:03 smithi019 ceph-mon[24574]: from='osd.6 [v2:172.21.15.172:6816/92131068,v1:172.21.15.172:6817/92131068]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-02-07T12:22:04.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:03 smithi019 ceph-mon[24574]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-02-07T12:22:04.605 DEBUG:teuthology.orchestra.run.smithi172:osd.6> sudo journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.6.service 2024-02-07T12:22:04.607 INFO:tasks.cephadm:Deploying osd.7 on smithi172 with /dev/vg_nvme/lv_1... 2024-02-07T12:22:04.608 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-02-07T12:22:05.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:04 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:05.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:04 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:22:05.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:04 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:05.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:04 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:22:05.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:04 smithi019 ceph-mon[24574]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-02-07T12:22:05.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:04 smithi019 ceph-mon[24574]: osdmap e44: 7 total, 6 up, 7 in 2024-02-07T12:22:05.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:04 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:22:05.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:04 smithi019 ceph-mon[24574]: from='osd.6 [v2:172.21.15.172:6816/92131068,v1:172.21.15.172:6817/92131068]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:22:05.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:04 smithi019 ceph-mon[24574]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:22:05.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:04 smithi019 ceph-mon[24574]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:22:05.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:04 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:05.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:04 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:22:05.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:04 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:05.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:04 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:22:05.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:04 smithi019 ceph-mon[29267]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-02-07T12:22:05.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:04 smithi019 ceph-mon[29267]: osdmap e44: 7 total, 6 up, 7 in 2024-02-07T12:22:05.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:04 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:22:05.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:04 smithi019 ceph-mon[29267]: from='osd.6 [v2:172.21.15.172:6816/92131068,v1:172.21.15.172:6817/92131068]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:22:05.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:04 smithi019 ceph-mon[29267]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:22:05.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:04 smithi019 ceph-mon[29267]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:22:05.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:04 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:05.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:04 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:22:05.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:04 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:05.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:04 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:22:05.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:04 smithi172 ceph-mon[25457]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-02-07T12:22:05.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:04 smithi172 ceph-mon[25457]: osdmap e44: 7 total, 6 up, 7 in 2024-02-07T12:22:05.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:04 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:22:05.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:04 smithi172 ceph-mon[25457]: from='osd.6 [v2:172.21.15.172:6816/92131068,v1:172.21.15.172:6817/92131068]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:22:05.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:04 smithi172 ceph-mon[25457]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:22:05.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:04 smithi172 ceph-mon[25457]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:22:05.370 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:22:05 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[34650]: 2024-02-07T12:22:05.043+0000 7f55f6ee2700 -1 osd.6 0 waiting for initial osdmap 2024-02-07T12:22:05.370 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:22:05 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[34650]: 2024-02-07T12:22:05.049+0000 7f55f387d700 -1 osd.6 45 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:22:06.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:06 smithi019 ceph-mon[24574]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]': finished 2024-02-07T12:22:06.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:06 smithi019 ceph-mon[24574]: osdmap e45: 7 total, 6 up, 7 in 2024-02-07T12:22:06.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:06 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:22:06.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:06 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:22:06.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:06 smithi019 ceph-mon[29267]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]': finished 2024-02-07T12:22:06.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:06 smithi019 ceph-mon[29267]: osdmap e45: 7 total, 6 up, 7 in 2024-02-07T12:22:06.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:06 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:22:06.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:06 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:22:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:06 smithi172 ceph-mon[25457]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]': finished 2024-02-07T12:22:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:06 smithi172 ceph-mon[25457]: osdmap e45: 7 total, 6 up, 7 in 2024-02-07T12:22:06.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:06 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:22:06.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:06 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:22:07.095 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:07 smithi172 ceph-mon[25457]: purged_snaps scrub starts 2024-02-07T12:22:07.095 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:07 smithi172 ceph-mon[25457]: purged_snaps scrub ok 2024-02-07T12:22:07.095 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:07 smithi172 ceph-mon[25457]: osd.6 [v2:172.21.15.172:6816/92131068,v1:172.21.15.172:6817/92131068] boot 2024-02-07T12:22:07.095 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:07 smithi172 ceph-mon[25457]: osdmap e46: 7 total, 7 up, 7 in 2024-02-07T12:22:07.095 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:07 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:22:07.096 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:07 smithi172 ceph-mon[25457]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:22:07.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:07 smithi019 ceph-mon[24574]: purged_snaps scrub starts 2024-02-07T12:22:07.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:07 smithi019 ceph-mon[24574]: purged_snaps scrub ok 2024-02-07T12:22:07.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:07 smithi019 ceph-mon[24574]: osd.6 [v2:172.21.15.172:6816/92131068,v1:172.21.15.172:6817/92131068] boot 2024-02-07T12:22:07.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:07 smithi019 ceph-mon[24574]: osdmap e46: 7 total, 7 up, 7 in 2024-02-07T12:22:07.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:07 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:22:07.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:07 smithi019 ceph-mon[24574]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:22:07.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:07 smithi019 ceph-mon[29267]: purged_snaps scrub starts 2024-02-07T12:22:07.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:07 smithi019 ceph-mon[29267]: purged_snaps scrub ok 2024-02-07T12:22:07.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:07 smithi019 ceph-mon[29267]: osd.6 [v2:172.21.15.172:6816/92131068,v1:172.21.15.172:6817/92131068] boot 2024-02-07T12:22:07.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:07 smithi019 ceph-mon[29267]: osdmap e46: 7 total, 7 up, 7 in 2024-02-07T12:22:07.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:07 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:22:07.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:07 smithi019 ceph-mon[29267]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:22:08.041 INFO:teuthology.orchestra.run.smithi172.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-02-07T12:22:08.041 INFO:teuthology.orchestra.run.smithi172.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-02-07T12:22:08.041 INFO:teuthology.orchestra.run.smithi172.stderr: stderr: 10+0 records in 2024-02-07T12:22:08.041 INFO:teuthology.orchestra.run.smithi172.stderr:10+0 records out 2024-02-07T12:22:08.041 INFO:teuthology.orchestra.run.smithi172.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0125035 s, 839 MB/s 2024-02-07T12:22:08.041 INFO:teuthology.orchestra.run.smithi172.stderr: stderr: 2024-02-07T12:22:08.042 INFO:teuthology.orchestra.run.smithi172.stderr:--> Zapping successful for: 2024-02-07T12:22:08.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:08 smithi019 ceph-mon[24574]: osdmap e47: 7 total, 7 up, 7 in 2024-02-07T12:22:08.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:08 smithi019 ceph-mon[29267]: osdmap e47: 7 total, 7 up, 7 in 2024-02-07T12:22:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:08 smithi172 ceph-mon[25457]: osdmap e47: 7 total, 7 up, 7 in 2024-02-07T12:22:08.737 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph orch daemon add osd smithi172:vg_nvme/lv_1 2024-02-07T12:22:09.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:09 smithi019 ceph-mon[24574]: osdmap e48: 7 total, 7 up, 7 in 2024-02-07T12:22:09.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:09 smithi019 ceph-mon[24574]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:22:09.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:09 smithi019 ceph-mon[29267]: osdmap e48: 7 total, 7 up, 7 in 2024-02-07T12:22:09.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:09 smithi019 ceph-mon[29267]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:22:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:09 smithi172 ceph-mon[25457]: osdmap e48: 7 total, 7 up, 7 in 2024-02-07T12:22:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:09 smithi172 ceph-mon[25457]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:22:10.323 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:10 smithi172 ceph-mon[25457]: osdmap e49: 7 total, 7 up, 7 in 2024-02-07T12:22:10.323 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:10 smithi172 ceph-mon[25457]: Detected new or changed devices on smithi172 2024-02-07T12:22:10.323 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:10 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:10.323 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:10 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:22:10.323 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:10 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:22:10.323 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:10 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:22:10.323 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:10 smithi172 ceph-mon[25457]: Adjusting osd_memory_target on smithi172 to 5696M 2024-02-07T12:22:10.324 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:10 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:10.324 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:10 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:10.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:10 smithi019 ceph-mon[24574]: osdmap e49: 7 total, 7 up, 7 in 2024-02-07T12:22:10.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:10 smithi019 ceph-mon[24574]: Detected new or changed devices on smithi172 2024-02-07T12:22:10.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:10 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:10.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:10 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:22:10.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:10 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:22:10.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:10 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:22:10.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:10 smithi019 ceph-mon[24574]: Adjusting osd_memory_target on smithi172 to 5696M 2024-02-07T12:22:10.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:10 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:10.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:10 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:10.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:10 smithi019 ceph-mon[29267]: osdmap e49: 7 total, 7 up, 7 in 2024-02-07T12:22:10.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:10 smithi019 ceph-mon[29267]: Detected new or changed devices on smithi172 2024-02-07T12:22:10.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:10 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:10.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:10 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:22:10.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:10 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:22:10.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:10 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:22:10.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:10 smithi019 ceph-mon[29267]: Adjusting osd_memory_target on smithi172 to 5696M 2024-02-07T12:22:10.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:10 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:10.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:10 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:11.233 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:11 smithi172 ceph-mon[25457]: osdmap e50: 7 total, 7 up, 7 in 2024-02-07T12:22:11.233 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:11 smithi172 ceph-mon[25457]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:22:11.233 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:11 smithi172 ceph-mon[25457]: from='client.24284 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi172:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:11.233 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:11 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:22:11.233 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:11 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:22:11.233 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:11 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:11.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:11 smithi019 ceph-mon[29267]: osdmap e50: 7 total, 7 up, 7 in 2024-02-07T12:22:11.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:11 smithi019 ceph-mon[29267]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:22:11.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:11 smithi019 ceph-mon[29267]: from='client.24284 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi172:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:11.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:11 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:22:11.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:11 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:22:11.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:11 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:11 smithi019 ceph-mon[24574]: osdmap e50: 7 total, 7 up, 7 in 2024-02-07T12:22:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:11 smithi019 ceph-mon[24574]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:22:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:11 smithi019 ceph-mon[24574]: from='client.24284 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi172:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:11 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:22:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:11 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:22:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:11 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:14.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:13 smithi019 ceph-mon[29267]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-02-07T12:22:14.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:13 smithi019 ceph-mon[29267]: from='client.? 172.21.15.172:0/4233430400' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c29a4703-29ad-4c37-a0cb-547860de3480"}]: dispatch 2024-02-07T12:22:14.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:13 smithi019 ceph-mon[29267]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c29a4703-29ad-4c37-a0cb-547860de3480"}]: dispatch 2024-02-07T12:22:14.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:13 smithi019 ceph-mon[29267]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c29a4703-29ad-4c37-a0cb-547860de3480"}]': finished 2024-02-07T12:22:14.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:13 smithi019 ceph-mon[29267]: osdmap e51: 8 total, 7 up, 8 in 2024-02-07T12:22:14.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:13 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:22:14.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:13 smithi019 ceph-mon[24574]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-02-07T12:22:14.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:13 smithi019 ceph-mon[24574]: from='client.? 172.21.15.172:0/4233430400' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c29a4703-29ad-4c37-a0cb-547860de3480"}]: dispatch 2024-02-07T12:22:14.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:13 smithi019 ceph-mon[24574]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c29a4703-29ad-4c37-a0cb-547860de3480"}]: dispatch 2024-02-07T12:22:14.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:13 smithi019 ceph-mon[24574]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c29a4703-29ad-4c37-a0cb-547860de3480"}]': finished 2024-02-07T12:22:14.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:13 smithi019 ceph-mon[24574]: osdmap e51: 8 total, 7 up, 8 in 2024-02-07T12:22:14.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:13 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:22:14.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:13 smithi172 ceph-mon[25457]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-02-07T12:22:14.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:13 smithi172 ceph-mon[25457]: from='client.? 172.21.15.172:0/4233430400' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c29a4703-29ad-4c37-a0cb-547860de3480"}]: dispatch 2024-02-07T12:22:14.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:13 smithi172 ceph-mon[25457]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c29a4703-29ad-4c37-a0cb-547860de3480"}]: dispatch 2024-02-07T12:22:14.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:13 smithi172 ceph-mon[25457]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c29a4703-29ad-4c37-a0cb-547860de3480"}]': finished 2024-02-07T12:22:14.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:13 smithi172 ceph-mon[25457]: osdmap e51: 8 total, 7 up, 8 in 2024-02-07T12:22:14.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:13 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:22:15.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:14 smithi019 ceph-mon[24574]: from='client.? 172.21.15.172:0/3219613411' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:22:15.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:14 smithi019 ceph-mon[29267]: from='client.? 172.21.15.172:0/3219613411' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:22:15.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:14 smithi172 ceph-mon[25457]: from='client.? 172.21.15.172:0/3219613411' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:22:16.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:15 smithi019 ceph-mon[24574]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-02-07T12:22:16.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:15 smithi019 ceph-mon[29267]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-02-07T12:22:16.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:15 smithi172 ceph-mon[25457]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-02-07T12:22:17.789 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:17 smithi172 ceph-mon[25457]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-02-07T12:22:18.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:17 smithi019 ceph-mon[24574]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-02-07T12:22:18.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:17 smithi019 ceph-mon[29267]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-02-07T12:22:19.991 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:19 smithi172 ceph-mon[25457]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-02-07T12:22:19.991 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:19 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-02-07T12:22:19.991 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:19 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:20.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:19 smithi019 ceph-mon[24574]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-02-07T12:22:20.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:19 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-02-07T12:22:20.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:19 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:20.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:19 smithi019 ceph-mon[29267]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-02-07T12:22:20.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:19 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-02-07T12:22:20.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:19 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:20.990 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:20 smithi172 ceph-mon[25457]: Deploying daemon osd.7 on smithi172 2024-02-07T12:22:21.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:20 smithi019 ceph-mon[29267]: Deploying daemon osd.7 on smithi172 2024-02-07T12:22:21.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:20 smithi019 ceph-mon[24574]: Deploying daemon osd.7 on smithi172 2024-02-07T12:22:21.976 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:21 smithi172 ceph-mon[25457]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-02-07T12:22:22.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:21 smithi019 ceph-mon[24574]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-02-07T12:22:22.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:21 smithi019 ceph-mon[29267]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-02-07T12:22:23.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:23 smithi172 ceph-mon[25457]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:22:24.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:23 smithi019 ceph-mon[24574]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:22:24.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:23 smithi019 ceph-mon[29267]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:22:25.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:25 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:25.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:25 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:22:25.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:25 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:25.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:25 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:22:25.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:25 smithi172 ceph-mon[25457]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:22:25.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:25 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:25.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:25 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:22:25.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:25 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:25.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:25 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:22:25.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:25 smithi019 ceph-mon[24574]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:22:25.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:25 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:25.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:25 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:22:25.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:25 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:25.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:25 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:22:25.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:25 smithi019 ceph-mon[29267]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:22:27.218 INFO:teuthology.orchestra.run.smithi172.stdout:Created osd(s) 7 on host 'smithi172' 2024-02-07T12:22:27.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:27 smithi172 ceph-mon[25457]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:22:27.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:27 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:27.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:27 smithi172 ceph-mon[25457]: from='osd.7 [v2:172.21.15.172:6824/1154934187,v1:172.21.15.172:6825/1154934187]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-07T12:22:27.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:27 smithi172 ceph-mon[25457]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-07T12:22:28.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:27 smithi019 ceph-mon[24574]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:22:28.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:27 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:28.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:27 smithi019 ceph-mon[24574]: from='osd.7 [v2:172.21.15.172:6824/1154934187,v1:172.21.15.172:6825/1154934187]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-07T12:22:28.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:27 smithi019 ceph-mon[24574]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-07T12:22:28.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:27 smithi019 ceph-mon[29267]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:22:28.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:27 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:28.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:27 smithi019 ceph-mon[29267]: from='osd.7 [v2:172.21.15.172:6824/1154934187,v1:172.21.15.172:6825/1154934187]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-07T12:22:28.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:27 smithi019 ceph-mon[29267]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-07T12:22:28.199 DEBUG:teuthology.orchestra.run.smithi172:osd.7> sudo journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.7.service 2024-02-07T12:22:28.201 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-02-07T12:22:28.201 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph osd stat -f json 2024-02-07T12:22:29.452 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:22:29 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[37568]: 2024-02-07T12:22:29.223+0000 7f11d24b7700 -1 osd.7 0 waiting for initial osdmap 2024-02-07T12:22:29.452 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:22:29 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[37568]: 2024-02-07T12:22:29.230+0000 7f11cee52700 -1 osd.7 53 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:22:29.453 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:29 smithi172 ceph-mon[25457]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-02-07T12:22:29.453 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:29 smithi172 ceph-mon[25457]: osdmap e52: 8 total, 7 up, 8 in 2024-02-07T12:22:29.453 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:29 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:22:29.453 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:29 smithi172 ceph-mon[25457]: from='osd.7 [v2:172.21.15.172:6824/1154934187,v1:172.21.15.172:6825/1154934187]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:22:29.453 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:29 smithi172 ceph-mon[25457]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:22:29.453 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:29 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:29.453 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:29 smithi172 ceph-mon[25457]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:22:29.453 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:29 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:22:29.453 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:29 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:22:29.559 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:29 smithi019 ceph-mon[24574]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-02-07T12:22:29.559 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:29 smithi019 ceph-mon[24574]: osdmap e52: 8 total, 7 up, 8 in 2024-02-07T12:22:29.559 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:29 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:22:29.560 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:29 smithi019 ceph-mon[24574]: from='osd.7 [v2:172.21.15.172:6824/1154934187,v1:172.21.15.172:6825/1154934187]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:22:29.560 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:29 smithi019 ceph-mon[24574]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:22:29.560 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:29 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:29.560 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:29 smithi019 ceph-mon[24574]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:22:29.560 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:29 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:22:29.560 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:29 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:22:29.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:29 smithi019 ceph-mon[29267]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-02-07T12:22:29.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:29 smithi019 ceph-mon[29267]: osdmap e52: 8 total, 7 up, 8 in 2024-02-07T12:22:29.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:29 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:22:29.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:29 smithi019 ceph-mon[29267]: from='osd.7 [v2:172.21.15.172:6824/1154934187,v1:172.21.15.172:6825/1154934187]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:22:29.560 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:29 smithi019 ceph-mon[29267]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:22:29.561 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:29 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:29.561 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:29 smithi019 ceph-mon[29267]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:22:29.561 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:29 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:22:29.561 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:29 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:22:30.310 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:22:30.342 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:30 smithi019 ceph-mon[29267]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]': finished 2024-02-07T12:22:30.342 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:30 smithi019 ceph-mon[29267]: osdmap e53: 8 total, 7 up, 8 in 2024-02-07T12:22:30.342 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:30 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:22:30.342 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:30 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:22:30.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:30 smithi019 ceph-mon[24574]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]': finished 2024-02-07T12:22:30.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:30 smithi019 ceph-mon[24574]: osdmap e53: 8 total, 7 up, 8 in 2024-02-07T12:22:30.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:30 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:22:30.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:30 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:22:30.483 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:30 smithi172 ceph-mon[25457]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]': finished 2024-02-07T12:22:30.483 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:30 smithi172 ceph-mon[25457]: osdmap e53: 8 total, 7 up, 8 in 2024-02-07T12:22:30.483 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:30 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:22:30.483 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:30 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:22:30.891 INFO:teuthology.orchestra.run.smithi019.stdout:{"epoch":54,"num_osds":8,"num_up_osds":8,"osd_up_since":1707308550,"num_in_osds":8,"osd_in_since":1707308533,"num_remapped_pgs":1} 2024-02-07T12:22:30.892 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph osd dump --format=json 2024-02-07T12:22:31.115 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/mon.a/config 2024-02-07T12:22:31.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:31 smithi019 ceph-mon[29267]: purged_snaps scrub starts 2024-02-07T12:22:31.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:31 smithi019 ceph-mon[29267]: purged_snaps scrub ok 2024-02-07T12:22:31.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:31 smithi019 ceph-mon[29267]: osd.7 [v2:172.21.15.172:6824/1154934187,v1:172.21.15.172:6825/1154934187] boot 2024-02-07T12:22:31.480 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:31 smithi019 ceph-mon[29267]: osdmap e54: 8 total, 8 up, 8 in 2024-02-07T12:22:31.480 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:31 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:22:31.480 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:31 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3113566598' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-02-07T12:22:31.480 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:31 smithi019 ceph-mon[29267]: pgmap v165: 1 pgs: 1 peering; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:31.480 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:31 smithi019 ceph-mon[29267]: osdmap e55: 8 total, 8 up, 8 in 2024-02-07T12:22:31.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:31 smithi019 ceph-mon[24574]: purged_snaps scrub starts 2024-02-07T12:22:31.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:31 smithi019 ceph-mon[24574]: purged_snaps scrub ok 2024-02-07T12:22:31.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:31 smithi019 ceph-mon[24574]: osd.7 [v2:172.21.15.172:6824/1154934187,v1:172.21.15.172:6825/1154934187] boot 2024-02-07T12:22:31.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:31 smithi019 ceph-mon[24574]: osdmap e54: 8 total, 8 up, 8 in 2024-02-07T12:22:31.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:31 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:22:31.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:31 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3113566598' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-02-07T12:22:31.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:31 smithi019 ceph-mon[24574]: pgmap v165: 1 pgs: 1 peering; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:31.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:31 smithi019 ceph-mon[24574]: osdmap e55: 8 total, 8 up, 8 in 2024-02-07T12:22:31.566 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:31 smithi172 ceph-mon[25457]: purged_snaps scrub starts 2024-02-07T12:22:31.566 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:31 smithi172 ceph-mon[25457]: purged_snaps scrub ok 2024-02-07T12:22:31.566 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:31 smithi172 ceph-mon[25457]: osd.7 [v2:172.21.15.172:6824/1154934187,v1:172.21.15.172:6825/1154934187] boot 2024-02-07T12:22:31.566 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:31 smithi172 ceph-mon[25457]: osdmap e54: 8 total, 8 up, 8 in 2024-02-07T12:22:31.566 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:31 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:22:31.566 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:31 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3113566598' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-02-07T12:22:31.566 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:31 smithi172 ceph-mon[25457]: pgmap v165: 1 pgs: 1 peering; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:31.566 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:31 smithi172 ceph-mon[25457]: osdmap e55: 8 total, 8 up, 8 in 2024-02-07T12:22:32.998 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:22:32.998 INFO:teuthology.orchestra.run.smithi019.stdout:{"epoch":57,"fsid":"ced2783c-c5b2-11ee-95b6-87774f69a715","created":"2024-02-07T12:17:38.251043+0000","modified":"2024-02-07T12:22:32.668412+0000","last_up_change":"2024-02-07T12:22:30.216345+0000","last_in_change":"2024-02-07T12:22:13.126065+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-02-07T12:20:34.945074+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"3ddfcd81-94a5-46f7-812e-c87f08345aef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6802","nonce":4112372591},{"type":"v1","addr":"172.21.15.19:6803","nonce":4112372591}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6804","nonce":4112372591},{"type":"v1","addr":"172.21.15.19:6805","nonce":4112372591}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6808","nonce":4112372591},{"type":"v1","addr":"172.21.15.19:6809","nonce":4112372591}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6806","nonce":4112372591},{"type":"v1","addr":"172.21.15.19:6807","nonce":4112372591}]},"public_addr":"172.21.15.19:6803/4112372591","cluster_addr":"172.21.15.19:6805/4112372591","heartbeat_back_addr":"172.21.15.19:6809/4112372591","heartbeat_front_addr":"172.21.15.19:6807/4112372591","state":["exists","up"]},{"osd":1,"uuid":"c5266822-bc44-4ede-90a8-bc3f880c1ea8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6810","nonce":3625191644},{"type":"v1","addr":"172.21.15.19:6811","nonce":3625191644}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6812","nonce":3625191644},{"type":"v1","addr":"172.21.15.19:6813","nonce":3625191644}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6816","nonce":3625191644},{"type":"v1","addr":"172.21.15.19:6817","nonce":3625191644}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6814","nonce":3625191644},{"type":"v1","addr":"172.21.15.19:6815","nonce":3625191644}]},"public_addr":"172.21.15.19:6811/3625191644","cluster_addr":"172.21.15.19:6813/3625191644","heartbeat_back_addr":"172.21.15.19:6817/3625191644","heartbeat_front_addr":"172.21.15.19:6815/3625191644","state":["exists","up"]},{"osd":2,"uuid":"1df65fd6-bed8-4ada-ba4c-7cde82b45a31","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6818","nonce":4106092366},{"type":"v1","addr":"172.21.15.19:6819","nonce":4106092366}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6820","nonce":4106092366},{"type":"v1","addr":"172.21.15.19:6821","nonce":4106092366}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6824","nonce":4106092366},{"type":"v1","addr":"172.21.15.19:6825","nonce":4106092366}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6822","nonce":4106092366},{"type":"v1","addr":"172.21.15.19:6823","nonce":4106092366}]},"public_addr":"172.21.15.19:6819/4106092366","cluster_addr":"172.21.15.19:6821/4106092366","heartbeat_back_addr":"172.21.15.19:6825/4106092366","heartbeat_front_addr":"172.21.15.19:6823/4106092366","state":["exists","up"]},{"osd":3,"uuid":"749f4475-2dbd-4166-b49b-3eaf63e1f7bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6826","nonce":2871829035},{"type":"v1","addr":"172.21.15.19:6827","nonce":2871829035}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6828","nonce":2871829035},{"type":"v1","addr":"172.21.15.19:6829","nonce":2871829035}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6832","nonce":2871829035},{"type":"v1","addr":"172.21.15.19:6833","nonce":2871829035}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6830","nonce":2871829035},{"type":"v1","addr":"172.21.15.19:6831","nonce":2871829035}]},"public_addr":"172.21.15.19:6827/2871829035","cluster_addr":"172.21.15.19:6829/2871829035","heartbeat_back_addr":"172.21.15.19:6833/2871829035","heartbeat_front_addr":"172.21.15.19:6831/2871829035","state":["exists","up"]},{"osd":4,"uuid":"7b6de577-7c35-4cc0-bf7e-cddbffbaad2b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6800","nonce":2804555213},{"type":"v1","addr":"172.21.15.172:6801","nonce":2804555213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6802","nonce":2804555213},{"type":"v1","addr":"172.21.15.172:6803","nonce":2804555213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6806","nonce":2804555213},{"type":"v1","addr":"172.21.15.172:6807","nonce":2804555213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6804","nonce":2804555213},{"type":"v1","addr":"172.21.15.172:6805","nonce":2804555213}]},"public_addr":"172.21.15.172:6801/2804555213","cluster_addr":"172.21.15.172:6803/2804555213","heartbeat_back_addr":"172.21.15.172:6807/2804555213","heartbeat_front_addr":"172.21.15.172:6805/2804555213","state":["exists","up"]},{"osd":5,"uuid":"efa49cf9-057d-4a6c-b3b8-368b9497f6ff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6808","nonce":3717679655},{"type":"v1","addr":"172.21.15.172:6809","nonce":3717679655}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6810","nonce":3717679655},{"type":"v1","addr":"172.21.15.172:6811","nonce":3717679655}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6814","nonce":3717679655},{"type":"v1","addr":"172.21.15.172:6815","nonce":3717679655}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6812","nonce":3717679655},{"type":"v1","addr":"172.21.15.172:6813","nonce":3717679655}]},"public_addr":"172.21.15.172:6809/3717679655","cluster_addr":"172.21.15.172:6811/3717679655","heartbeat_back_addr":"172.21.15.172:6815/3717679655","heartbeat_front_addr":"172.21.15.172:6813/3717679655","state":["exists","up"]},{"osd":6,"uuid":"fe43b6a2-9866-4da0-a167-f4ba5564cdbc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6816","nonce":92131068},{"type":"v1","addr":"172.21.15.172:6817","nonce":92131068}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6818","nonce":92131068},{"type":"v1","addr":"172.21.15.172:6819","nonce":92131068}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6822","nonce":92131068},{"type":"v1","addr":"172.21.15.172:6823","nonce":92131068}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6820","nonce":92131068},{"type":"v1","addr":"172.21.15.172:6821","nonce":92131068}]},"public_addr":"172.21.15.172:6817/92131068","cluster_addr":"172.21.15.172:6819/92131068","heartbeat_back_addr":"172.21.15.172:6823/92131068","heartbeat_front_addr":"172.21.15.172:6821/92131068","state":["exists","up"]},{"osd":7,"uuid":"c29a4703-29ad-4c37-a0cb-547860de3480","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6824","nonce":1154934187},{"type":"v1","addr":"172.21.15.172:6825","nonce":1154934187}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6826","nonce":1154934187},{"type":"v1","addr":"172.21.15.172:6827","nonce":1154934187}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6830","nonce":1154934187},{"type":"v1","addr":"172.21.15.172:6831","nonce":1154934187}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6828","nonce":1154934187},{"type":"v1","addr":"172.21.15.172:6829","nonce":1154934187}]},"public_addr":"172.21.15.172:6825/1154934187","cluster_addr":"172.21.15.172:6827/1154934187","heartbeat_back_addr":"172.21.15.172:6831/1154934187","heartbeat_front_addr":"172.21.15.172:6829/1154934187","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:19:46.348476+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:20:09.908222+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:20:33.189734+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:20:55.901187+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:21:17.030966+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:21:40.885450+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:22:04.445626+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:22:28.526412+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.19:0/2628806613":"2024-02-08T12:18:27.833418+0000","172.21.15.19:6800/2471401560":"2024-02-08T12:18:27.833418+0000","172.21.15.19:0/784284954":"2024-02-08T12:18:27.833418+0000","172.21.15.19:6800/2571038478":"2024-02-08T12:18:05.905616+0000","172.21.15.19:6801/2571038478":"2024-02-08T12:18:05.905616+0000","172.21.15.19:6801/2471401560":"2024-02-08T12:18:27.833418+0000","172.21.15.19:0/1231318565":"2024-02-08T12:18:05.905616+0000","172.21.15.19:0/2080165887":"2024-02-08T12:18:27.833418+0000","172.21.15.19:0/1819641709":"2024-02-08T12:18:05.905616+0000","172.21.15.19:0/4109166938":"2024-02-08T12:18:05.905616+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-02-07T12:22:33.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[24574]: Detected new or changed devices on smithi172 2024-02-07T12:22:33.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:33.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:22:33.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:22:33.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:22:33.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:22:33.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:33.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:33.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:22:33.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:33.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:22:33.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:33.249 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[24574]: osdmap e56: 8 total, 8 up, 8 in 2024-02-07T12:22:33.249 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[24574]: osdmap e57: 8 total, 8 up, 8 in 2024-02-07T12:22:33.249 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[29267]: Detected new or changed devices on smithi172 2024-02-07T12:22:33.249 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:33.249 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:22:33.249 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:22:33.249 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:22:33.249 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:22:33.249 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:33.249 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:33.249 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:22:33.249 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:33.250 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:22:33.250 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:33.250 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[29267]: osdmap e56: 8 total, 8 up, 8 in 2024-02-07T12:22:33.250 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:32 smithi019 ceph-mon[29267]: osdmap e57: 8 total, 8 up, 8 in 2024-02-07T12:22:33.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:32 smithi172 ceph-mon[25457]: Detected new or changed devices on smithi172 2024-02-07T12:22:33.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:32 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:33.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:32 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:22:33.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:32 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:22:33.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:32 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:22:33.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:32 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:22:33.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:32 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:33.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:32 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:33.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:32 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:22:33.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:32 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:33.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:32 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:22:33.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:32 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:33.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:32 smithi172 ceph-mon[25457]: osdmap e56: 8 total, 8 up, 8 in 2024-02-07T12:22:33.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:32 smithi172 ceph-mon[25457]: osdmap e57: 8 total, 8 up, 8 in 2024-02-07T12:22:33.733 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-02-07T12:20:34.945074+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-02-07T12:22:33.733 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph osd pool get .mgr pg_num 2024-02-07T12:22:33.954 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/mon.a/config 2024-02-07T12:22:34.076 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:33 smithi019 ceph-mon[24574]: Adjusting osd_memory_target on smithi172 to 4272M 2024-02-07T12:22:34.076 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:33 smithi019 ceph-mon[24574]: pgmap v168: 1 pgs: 1 peering; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:34.076 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:33 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1906998571' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-02-07T12:22:34.076 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:33 smithi019 ceph-mon[24574]: osdmap e58: 8 total, 8 up, 8 in 2024-02-07T12:22:34.076 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:33 smithi019 ceph-mon[29267]: Adjusting osd_memory_target on smithi172 to 4272M 2024-02-07T12:22:34.076 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:33 smithi019 ceph-mon[29267]: pgmap v168: 1 pgs: 1 peering; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:34.077 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:33 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1906998571' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-02-07T12:22:34.077 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:33 smithi019 ceph-mon[29267]: osdmap e58: 8 total, 8 up, 8 in 2024-02-07T12:22:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:33 smithi172 ceph-mon[25457]: Adjusting osd_memory_target on smithi172 to 4272M 2024-02-07T12:22:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:33 smithi172 ceph-mon[25457]: pgmap v168: 1 pgs: 1 peering; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:33 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1906998571' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-02-07T12:22:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:33 smithi172 ceph-mon[25457]: osdmap e58: 8 total, 8 up, 8 in 2024-02-07T12:22:35.284 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:34 smithi019 ceph-mon[24574]: pgmap v171: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:35.284 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:34 smithi019 ceph-mon[29267]: pgmap v171: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:34 smithi172 ceph-mon[25457]: pgmap v171: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:35.814 INFO:teuthology.orchestra.run.smithi019.stdout:pg_num: 1 2024-02-07T12:22:36.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:35 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/913122537' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-02-07T12:22:36.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:35 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/913122537' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-02-07T12:22:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:35 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/913122537' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-02-07T12:22:36.522 INFO:tasks.cephadm:Adding prometheus.a on smithi172 2024-02-07T12:22:36.523 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph orch apply prometheus '1;smithi172=a' 2024-02-07T12:22:37.115 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:36 smithi172 ceph-mon[25457]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 95 KiB/s, 0 objects/s recovering 2024-02-07T12:22:37.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:36 smithi019 ceph-mon[24574]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 95 KiB/s, 0 objects/s recovering 2024-02-07T12:22:37.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:36 smithi019 ceph-mon[29267]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 95 KiB/s, 0 objects/s recovering 2024-02-07T12:22:38.525 INFO:teuthology.orchestra.run.smithi172.stdout:Scheduled prometheus update... 2024-02-07T12:22:39.115 DEBUG:teuthology.orchestra.run.smithi172:prometheus.a> sudo journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@prometheus.a.service 2024-02-07T12:22:39.117 INFO:tasks.cephadm:Adding node-exporter.a on smithi019 2024-02-07T12:22:39.117 INFO:tasks.cephadm:Adding node-exporter.b on smithi172 2024-02-07T12:22:39.118 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph orch apply node-exporter '2;smithi019=a;smithi172=b' 2024-02-07T12:22:39.622 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:39 smithi172 ceph-mon[25457]: from='client.24323 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi172=a", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:39.622 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:39 smithi172 ceph-mon[25457]: Saving service prometheus spec with placement smithi172=a;count:1 2024-02-07T12:22:39.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:39 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:39.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:39 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:22:39.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:39 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:39.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:39 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:22:39.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:39 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:39.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:39 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-02-07T12:22:39.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:39 smithi172 ceph-mon[25457]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-02-07T12:22:39.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:39 smithi019 ceph-mon[24574]: from='client.24323 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi172=a", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:39.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:39 smithi019 ceph-mon[24574]: Saving service prometheus spec with placement smithi172=a;count:1 2024-02-07T12:22:39.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:39 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:39.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:39 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:22:39.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:39 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:39.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:39 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:22:39.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:39 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:39.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:39 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-02-07T12:22:39.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:39 smithi019 ceph-mon[24574]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-02-07T12:22:39.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ignoring --setuser ceph since I am not root 2024-02-07T12:22:39.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ignoring --setgroup ceph since I am not root 2024-02-07T12:22:39.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:39 smithi019 ceph-mgr[24810]: -- 172.21.15.19:0/3481451001 <== mon.0 v2:172.21.15.19:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56535eeb4340 con 0x56535ee8c800 2024-02-07T12:22:39.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:39 smithi019 ceph-mon[29267]: from='client.24323 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi172=a", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:39.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:39 smithi019 ceph-mon[29267]: Saving service prometheus spec with placement smithi172=a;count:1 2024-02-07T12:22:39.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:39 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:39.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:39 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:22:39.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:39 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:39.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:39 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:22:39.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:39 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' 2024-02-07T12:22:39.847 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:39 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-02-07T12:22:39.847 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:39 smithi019 ceph-mon[29267]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-02-07T12:22:40.087 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:39 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ignoring --setuser ceph since I am not root 2024-02-07T12:22:40.088 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:39 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ignoring --setgroup ceph since I am not root 2024-02-07T12:22:40.088 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:39 smithi172 ceph-mgr[26644]: -- 172.21.15.172:0/1586603881 <== mon.2 v2:172.21.15.172:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x558df627c340 con 0x558df6255000 2024-02-07T12:22:40.088 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:40 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:22:40.084+0000 7f7cc270c000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-02-07T12:22:40.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:40 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:22:40.173+0000 7f7cc270c000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-02-07T12:22:40.579 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:40 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:22:40.094+0000 7f9ce08a8000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-02-07T12:22:40.579 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:40 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:22:40.185+0000 7f9ce08a8000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-02-07T12:22:40.736 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:40 smithi172 ceph-mon[25457]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-02-07T12:22:40.737 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:40 smithi172 ceph-mon[25457]: mgrmap e16: y(active, since 4m), standbys: x 2024-02-07T12:22:40.844 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:40 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:22:40.797+0000 7f9ce08a8000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-02-07T12:22:40.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:40 smithi019 ceph-mon[24574]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-02-07T12:22:40.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:40 smithi019 ceph-mon[24574]: mgrmap e16: y(active, since 4m), standbys: x 2024-02-07T12:22:40.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:40 smithi019 ceph-mon[29267]: from='mgr.14150 172.21.15.19:0/1873555618' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-02-07T12:22:40.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:40 smithi019 ceph-mon[29267]: mgrmap e16: y(active, since 4m), standbys: x 2024-02-07T12:22:41.114 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:40 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:22:40.779+0000 7f7cc270c000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-02-07T12:22:41.115 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:40 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:22:40.869+0000 7f7cc270c000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-02-07T12:22:41.115 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:40 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:22:40.955+0000 7f7cc270c000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-02-07T12:22:41.115 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:41 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:22:41.111+0000 7f7cc270c000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-02-07T12:22:41.215 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:40 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:22:40.888+0000 7f9ce08a8000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-02-07T12:22:41.215 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:40 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:22:40.974+0000 7f9ce08a8000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-02-07T12:22:41.216 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:22:41.127+0000 7f9ce08a8000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-02-07T12:22:41.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:41 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:22:41.193+0000 7f7cc270c000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-02-07T12:22:41.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:41 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:22:41.302+0000 7f7cc270c000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-02-07T12:22:41.581 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:22:41.209+0000 7f9ce08a8000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-02-07T12:22:41.581 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:22:41.323+0000 7f9ce08a8000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-02-07T12:22:41.581 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:22:41.411+0000 7f9ce08a8000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-02-07T12:22:41.844 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:22:41.577+0000 7f9ce08a8000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-02-07T12:22:41.844 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:22:41.665+0000 7f9ce08a8000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-02-07T12:22:41.869 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:41 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:22:41.384+0000 7f7cc270c000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-02-07T12:22:41.869 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:41 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:22:41.548+0000 7f7cc270c000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-02-07T12:22:41.869 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:41 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:22:41.632+0000 7f7cc270c000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-02-07T12:22:42.965 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:42 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:22:42.645+0000 7f7cc270c000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-02-07T12:22:42.965 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:42 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:22:42.774+0000 7f7cc270c000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-02-07T12:22:43.055 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:42 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:22:42.723+0000 7f9ce08a8000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-02-07T12:22:43.056 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:42 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:22:42.857+0000 7f9ce08a8000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-02-07T12:22:43.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:22:43.053+0000 7f9ce08a8000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-02-07T12:22:43.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:22:43.143+0000 7f9ce08a8000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-02-07T12:22:43.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:22:43.284+0000 7f9ce08a8000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:22:43.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:42 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:22:42.961+0000 7f7cc270c000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-02-07T12:22:43.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:43 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:22:43.049+0000 7f7cc270c000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-02-07T12:22:43.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:43 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:22:43.182+0000 7f7cc270c000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:22:43.804 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:43 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:22:43.500+0000 7f7cc270c000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-02-07T12:22:43.804 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:43 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:22:43.717+0000 7f7cc270c000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:22:43.928 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:22:43.610+0000 7f9ce08a8000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-02-07T12:22:43.929 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:22:43.838+0000 7f9ce08a8000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:22:44.119 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:43 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:22:43.801+0000 7f7cc270c000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-02-07T12:22:44.119 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:43 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:22:43.968+0000 7f7cc270c000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-02-07T12:22:44.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:22:43.926+0000 7f9ce08a8000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-02-07T12:22:44.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:44 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:22:44.102+0000 7f9ce08a8000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-02-07T12:22:45.167 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:44 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:22:44.876+0000 7f7cc270c000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-02-07T12:22:45.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:45 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:22:45.050+0000 7f9ce08a8000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-02-07T12:22:45.423 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:45 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:22:45.164+0000 7f7cc270c000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-02-07T12:22:45.423 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:45 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:22:45.258+0000 7f7cc270c000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-02-07T12:22:45.423 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:45 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:22:45.405+0000 7f7cc270c000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-02-07T12:22:45.424 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:45 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: [07/Feb/2024:12:22:45] ENGINE Bus STARTING 2024-02-07T12:22:45.618 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:45 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:22:45.359+0000 7f9ce08a8000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-02-07T12:22:45.618 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:45 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:22:45.457+0000 7f9ce08a8000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-02-07T12:22:45.618 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:45 smithi019 ceph-mon[24574]: Standby manager daemon x restarted 2024-02-07T12:22:45.618 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:45 smithi019 ceph-mon[24574]: Standby manager daemon x started 2024-02-07T12:22:45.618 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:45 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.172:0/2901664140' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-02-07T12:22:45.618 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:45 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.172:0/2901664140' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:22:45.618 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:45 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.172:0/2901664140' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-02-07T12:22:45.618 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:45 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.172:0/2901664140' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:22:45.619 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:45 smithi019 ceph-mon[29267]: Standby manager daemon x restarted 2024-02-07T12:22:45.619 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:45 smithi019 ceph-mon[29267]: Standby manager daemon x started 2024-02-07T12:22:45.619 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:45 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/2901664140' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-02-07T12:22:45.619 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:45 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/2901664140' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:22:45.619 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:45 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/2901664140' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-02-07T12:22:45.619 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:45 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/2901664140' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:22:45.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:45 smithi172 ceph-mon[25457]: Standby manager daemon x restarted 2024-02-07T12:22:45.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:45 smithi172 ceph-mon[25457]: Standby manager daemon x started 2024-02-07T12:22:45.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:45 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.172:0/2901664140' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-02-07T12:22:45.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:45 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.172:0/2901664140' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:22:45.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:45 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.172:0/2901664140' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-02-07T12:22:45.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:45 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.172:0/2901664140' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:22:45.870 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:45 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: CherryPy Checker: 2024-02-07T12:22:45.870 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:45 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: The Application mounted at '' has an empty config. 2024-02-07T12:22:45.870 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:45 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: 2024-02-07T12:22:45.870 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:45 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: [07/Feb/2024:12:22:45] ENGINE Serving on http://:::9283 2024-02-07T12:22:45.870 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:22:45 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: [07/Feb/2024:12:22:45] ENGINE Bus STARTED 2024-02-07T12:22:46.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:45 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:22:45.615+0000 7f9ce08a8000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-02-07T12:22:46.828 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:46 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: [07/Feb/2024:12:22:46] ENGINE Bus STARTING 2024-02-07T12:22:46.828 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:46 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: CherryPy Checker: 2024-02-07T12:22:46.828 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:46 smithi019 ceph-mon[29267]: mgrmap e17: y(active, since 4m), standbys: x 2024-02-07T12:22:46.828 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:46 smithi019 ceph-mon[29267]: Active manager daemon y restarted 2024-02-07T12:22:46.829 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:46 smithi019 ceph-mon[29267]: Activating manager daemon y 2024-02-07T12:22:46.829 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:46 smithi019 ceph-mon[29267]: osdmap e59: 8 total, 8 up, 8 in 2024-02-07T12:22:46.829 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:46 smithi019 ceph-mon[24574]: mgrmap e17: y(active, since 4m), standbys: x 2024-02-07T12:22:46.829 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:46 smithi019 ceph-mon[24574]: Active manager daemon y restarted 2024-02-07T12:22:46.829 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:46 smithi019 ceph-mon[24574]: Activating manager daemon y 2024-02-07T12:22:46.829 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:46 smithi019 ceph-mon[24574]: osdmap e59: 8 total, 8 up, 8 in 2024-02-07T12:22:46.836 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:46 smithi172 ceph-mon[25457]: mgrmap e17: y(active, since 4m), standbys: x 2024-02-07T12:22:46.836 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:46 smithi172 ceph-mon[25457]: Active manager daemon y restarted 2024-02-07T12:22:46.836 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:46 smithi172 ceph-mon[25457]: Activating manager daemon y 2024-02-07T12:22:46.836 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:46 smithi172 ceph-mon[25457]: osdmap e59: 8 total, 8 up, 8 in 2024-02-07T12:22:47.082 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:46 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: The Application mounted at '' has an empty config. 2024-02-07T12:22:47.082 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:46 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:22:47.083 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:46 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: [07/Feb/2024:12:22:46] ENGINE Serving on http://:::9283 2024-02-07T12:22:47.083 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:46 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: [07/Feb/2024:12:22:46] ENGINE Bus STARTED 2024-02-07T12:22:47.510 INFO:teuthology.orchestra.run.smithi172.stdout:Scheduled node-exporter update... 2024-02-07T12:22:47.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[29267]: mgrmap e18: y(active, starting, since 0.861512s), standbys: x 2024-02-07T12:22:47.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:22:47.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:22:47.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:22:47.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-02-07T12:22:47.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-02-07T12:22:47.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:22:47.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:22:47.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:22:47.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:22:47.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:22:47.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:22:47.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:22:47.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:22:47.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:22:47.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:22:47.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:22:47.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[29267]: Manager daemon y is now available 2024-02-07T12:22:47.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:22:47.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:22:47.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:47.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:22:47.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:47.847 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:22:47.847 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:22:47.847 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:22:47.847 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:22:47.847 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[24574]: mgrmap e18: y(active, starting, since 0.861512s), standbys: x 2024-02-07T12:22:47.847 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:22:47.847 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:22:47.847 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:22:47.847 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-02-07T12:22:47.847 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-02-07T12:22:47.847 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:22:47.847 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:22:47.848 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:22:47.848 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:22:47.848 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:22:47.848 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:22:47.848 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:22:47.848 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:22:47.848 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:22:47.848 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:22:47.848 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:22:47.848 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[24574]: Manager daemon y is now available 2024-02-07T12:22:47.848 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:22:47.848 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:22:47.848 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:47.848 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:22:47.849 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:47.849 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:22:47.849 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:22:47.849 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:22:47.849 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:47 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:22:47.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:47 smithi172 ceph-mon[25457]: mgrmap e18: y(active, starting, since 0.861512s), standbys: x 2024-02-07T12:22:47.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:22:47.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:22:47.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:22:47.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-02-07T12:22:47.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-02-07T12:22:47.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:22:47.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:22:47.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:22:47.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:22:47.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:22:47.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:22:47.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:22:47.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:22:47.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:22:47.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:22:47.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:22:47.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:47 smithi172 ceph-mon[25457]: Manager daemon y is now available 2024-02-07T12:22:47.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:22:47.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:22:47.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:47.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:22:47.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:47 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:47.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:22:47.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:47 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:22:47.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:22:47.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:47 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:22:48.507 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:48 smithi019 ceph-mon[24574]: mgrmap e19: y(active, since 1.86531s), standbys: x 2024-02-07T12:22:48.507 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:48 smithi019 ceph-mon[24574]: Saving service node-exporter spec with placement smithi019=a;smithi172=b;count:2 2024-02-07T12:22:48.507 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:48 smithi019 ceph-mon[24574]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:48.507 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:48 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:48.507 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:48 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:48.601 DEBUG:teuthology.orchestra.run.smithi019:node-exporter.a> sudo journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@node-exporter.a.service 2024-02-07T12:22:48.604 DEBUG:teuthology.orchestra.run.smithi172:node-exporter.b> sudo journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@node-exporter.b.service 2024-02-07T12:22:48.606 INFO:tasks.cephadm:Adding alertmanager.a on smithi019 2024-02-07T12:22:48.606 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph orch apply alertmanager '1;smithi019=a' 2024-02-07T12:22:48.792 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:48 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: [07/Feb/2024:12:22:48] ENGINE Bus STARTING 2024-02-07T12:22:48.793 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:48 smithi019 ceph-mon[29267]: mgrmap e19: y(active, since 1.86531s), standbys: x 2024-02-07T12:22:48.793 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:48 smithi019 ceph-mon[29267]: Saving service node-exporter spec with placement smithi019=a;smithi172=b;count:2 2024-02-07T12:22:48.793 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:48 smithi019 ceph-mon[29267]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:48.793 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:48 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:48.793 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:48 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:48.801 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:48 smithi172 ceph-mon[25457]: mgrmap e19: y(active, since 1.86531s), standbys: x 2024-02-07T12:22:48.801 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:48 smithi172 ceph-mon[25457]: Saving service node-exporter spec with placement smithi019=a;smithi172=b;count:2 2024-02-07T12:22:48.801 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:48 smithi172 ceph-mon[25457]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:48.801 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:48 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:48.801 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:48 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:49.048 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:48 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: [07/Feb/2024:12:22:48] ENGINE Serving on https://172.21.15.19:7150 2024-02-07T12:22:49.049 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:22:48 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: [07/Feb/2024:12:22:48] ENGINE Bus STARTED 2024-02-07T12:22:49.805 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:49 smithi172 ceph-mon[25457]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:49.805 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:49 smithi172 ceph-mon[25457]: mgrmap e20: y(active, since 2s), standbys: x 2024-02-07T12:22:49.805 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:49 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:49.806 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:49 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:49.806 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:49 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:49.806 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:49 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:49.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:49 smithi019 ceph-mon[29267]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:49.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:49 smithi019 ceph-mon[29267]: mgrmap e20: y(active, since 2s), standbys: x 2024-02-07T12:22:49.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:49 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:49.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:49 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:49.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:49 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:49.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:49 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:49.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:49 smithi019 ceph-mon[24574]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:49.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:49 smithi019 ceph-mon[24574]: mgrmap e20: y(active, since 2s), standbys: x 2024-02-07T12:22:49.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:49 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:49.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:49 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:49.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:49 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:49.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:49 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:50.763 INFO:teuthology.orchestra.run.smithi172.stdout:Scheduled alertmanager update... 2024-02-07T12:22:50.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:50 smithi019 ceph-mon[24574]: [07/Feb/2024:12:22:48] ENGINE Bus STARTING 2024-02-07T12:22:50.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:50 smithi019 ceph-mon[24574]: Updating smithi019:/etc/ceph/ceph.conf 2024-02-07T12:22:50.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:50 smithi019 ceph-mon[24574]: [07/Feb/2024:12:22:48] ENGINE Serving on https://172.21.15.19:7150 2024-02-07T12:22:50.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:50 smithi019 ceph-mon[24574]: [07/Feb/2024:12:22:48] ENGINE Bus STARTED 2024-02-07T12:22:50.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:50 smithi019 ceph-mon[24574]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:22:50.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:50 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:50.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:50 smithi019 ceph-mon[24574]: Updating smithi172:/etc/ceph/ceph.conf 2024-02-07T12:22:50.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:50 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:50.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:50 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:50.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:50 smithi019 ceph-mon[29267]: [07/Feb/2024:12:22:48] ENGINE Bus STARTING 2024-02-07T12:22:50.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:50 smithi019 ceph-mon[29267]: Updating smithi019:/etc/ceph/ceph.conf 2024-02-07T12:22:50.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:50 smithi019 ceph-mon[29267]: [07/Feb/2024:12:22:48] ENGINE Serving on https://172.21.15.19:7150 2024-02-07T12:22:50.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:50 smithi019 ceph-mon[29267]: [07/Feb/2024:12:22:48] ENGINE Bus STARTED 2024-02-07T12:22:50.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:50 smithi019 ceph-mon[29267]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:22:50.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:50 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:50.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:50 smithi019 ceph-mon[29267]: Updating smithi172:/etc/ceph/ceph.conf 2024-02-07T12:22:50.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:50 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:50.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:50 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:50.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:50 smithi172 ceph-mon[25457]: [07/Feb/2024:12:22:48] ENGINE Bus STARTING 2024-02-07T12:22:50.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:50 smithi172 ceph-mon[25457]: Updating smithi019:/etc/ceph/ceph.conf 2024-02-07T12:22:50.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:50 smithi172 ceph-mon[25457]: [07/Feb/2024:12:22:48] ENGINE Serving on https://172.21.15.19:7150 2024-02-07T12:22:50.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:50 smithi172 ceph-mon[25457]: [07/Feb/2024:12:22:48] ENGINE Bus STARTED 2024-02-07T12:22:50.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:50 smithi172 ceph-mon[25457]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:22:50.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:50 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:50.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:50 smithi172 ceph-mon[25457]: Updating smithi172:/etc/ceph/ceph.conf 2024-02-07T12:22:50.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:50 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:50.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:50 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:51.498 DEBUG:teuthology.orchestra.run.smithi019:alertmanager.a> sudo journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@alertmanager.a.service 2024-02-07T12:22:51.501 INFO:tasks.cephadm:Adding grafana.a on smithi172 2024-02-07T12:22:51.501 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph orch apply grafana '1;smithi172=a' 2024-02-07T12:22:51.813 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:51 smithi172 ceph-mon[25457]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:22:51.813 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:51 smithi172 ceph-mon[25457]: Deploying daemon prometheus.a on smithi172 2024-02-07T12:22:51.813 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:51 smithi172 ceph-mon[25457]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:51.813 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:51 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:51.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:51 smithi019 ceph-mon[24574]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:22:51.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:51 smithi019 ceph-mon[24574]: Deploying daemon prometheus.a on smithi172 2024-02-07T12:22:51.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:51 smithi019 ceph-mon[24574]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:51.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:51 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:51.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:51 smithi019 ceph-mon[29267]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:22:51.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:51 smithi019 ceph-mon[29267]: Deploying daemon prometheus.a on smithi172 2024-02-07T12:22:51.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:51 smithi019 ceph-mon[29267]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:51.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:51 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:52.765 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:52 smithi172 ceph-mon[25457]: from='client.24368 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi019=a", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:52.766 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:52 smithi172 ceph-mon[25457]: Saving service alertmanager spec with placement smithi019=a;count:1 2024-02-07T12:22:52.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:52 smithi019 ceph-mon[29267]: from='client.24368 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi019=a", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:52.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:52 smithi019 ceph-mon[29267]: Saving service alertmanager spec with placement smithi019=a;count:1 2024-02-07T12:22:52.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:52 smithi019 ceph-mon[24574]: from='client.24368 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi019=a", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:52.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:52 smithi019 ceph-mon[24574]: Saving service alertmanager spec with placement smithi019=a;count:1 2024-02-07T12:22:53.798 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:53 smithi172 ceph-mon[25457]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:53.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:53 smithi019 ceph-mon[24574]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:53.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:53 smithi019 ceph-mon[29267]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:54.137 INFO:teuthology.orchestra.run.smithi172.stdout:Scheduled grafana update... 2024-02-07T12:22:55.113 DEBUG:teuthology.orchestra.run.smithi172:grafana.a> sudo journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@grafana.a.service 2024-02-07T12:22:55.116 INFO:tasks.cephadm:Setting up client nodes... 2024-02-07T12:22:55.119 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-02-07T12:22:55.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:55 smithi172 ceph-mon[25457]: from='client.24374 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi172=a", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:55.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:55 smithi172 ceph-mon[25457]: Saving service grafana spec with placement smithi172=a;count:1 2024-02-07T12:22:55.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:55 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:55.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:55 smithi172 ceph-mon[25457]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:55.438 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:55 smithi019 ceph-mon[29267]: from='client.24374 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi172=a", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:55.438 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:55 smithi019 ceph-mon[29267]: Saving service grafana spec with placement smithi172=a;count:1 2024-02-07T12:22:55.438 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:55 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:55.438 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:55 smithi019 ceph-mon[29267]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:55.439 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:55 smithi019 ceph-mon[24574]: from='client.24374 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi172=a", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:55.439 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:55 smithi019 ceph-mon[24574]: Saving service grafana spec with placement smithi172=a;count:1 2024-02-07T12:22:55.439 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:55 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:22:55.439 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:55 smithi019 ceph-mon[24574]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:57.365 INFO:teuthology.orchestra.run.smithi019.stdout:[client.0] 2024-02-07T12:22:57.365 INFO:teuthology.orchestra.run.smithi019.stdout: key = AQAhdsNl3QhxFRAAGPvu0XBAC/QoAn9/hhQUfg== 2024-02-07T12:22:57.636 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:57 smithi019 ceph-mon[24574]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:57.637 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:57 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3006228163' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-02-07T12:22:57.637 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:57 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3006228163' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-02-07T12:22:57.637 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:57 smithi019 ceph-mon[29267]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:57.637 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:57 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3006228163' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-02-07T12:22:57.637 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:57 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3006228163' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-02-07T12:22:57.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:57 smithi172 ceph-mon[25457]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:57.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:57 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3006228163' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-02-07T12:22:57.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:57 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3006228163' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-02-07T12:22:57.994 DEBUG:teuthology.orchestra.run.smithi019:> set -ex 2024-02-07T12:22:57.994 DEBUG:teuthology.orchestra.run.smithi019:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-02-07T12:22:57.994 DEBUG:teuthology.orchestra.run.smithi019:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-02-07T12:22:58.037 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-02-07T12:22:58.854 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:22:58 smithi172 ceph-mon[25457]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:59.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:22:58 smithi019 ceph-mon[24574]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:22:59.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:22:58 smithi019 ceph-mon[29267]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:01.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:01 smithi172 ceph-mon[25457]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:01.811 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:01 smithi019 ceph-mon[29267]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:01.811 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:01 smithi019 ceph-mon[24574]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:01.835 INFO:teuthology.orchestra.run.smithi172.stdout:[client.1] 2024-02-07T12:23:01.835 INFO:teuthology.orchestra.run.smithi172.stdout: key = AQAldsNlAAc9MRAA1pbyriOTxn5so9c1YPosQA== 2024-02-07T12:23:02.424 DEBUG:teuthology.orchestra.run.smithi172:> set -ex 2024-02-07T12:23:02.424 DEBUG:teuthology.orchestra.run.smithi172:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-02-07T12:23:02.424 DEBUG:teuthology.orchestra.run.smithi172:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-02-07T12:23:02.466 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-02-07T12:23:02.466 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-02-07T12:23:02.466 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph mgr dump --format=json 2024-02-07T12:23:02.689 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/mon.a/config 2024-02-07T12:23:02.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:02 smithi172 ceph-mon[25457]: from='client.? 172.21.15.172:0/3017354552' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-02-07T12:23:02.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:02 smithi172 ceph-mon[25457]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-02-07T12:23:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:02 smithi172 ceph-mon[25457]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-02-07T12:23:02.811 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:02 smithi019 ceph-mon[29267]: from='client.? 172.21.15.172:0/3017354552' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-02-07T12:23:02.811 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:02 smithi019 ceph-mon[29267]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-02-07T12:23:02.811 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:02 smithi019 ceph-mon[29267]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-02-07T12:23:02.811 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:02 smithi019 ceph-mon[24574]: from='client.? 172.21.15.172:0/3017354552' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-02-07T12:23:02.811 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:02 smithi019 ceph-mon[24574]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-02-07T12:23:02.811 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:02 smithi019 ceph-mon[24574]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-02-07T12:23:03.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:03 smithi172 ceph-mon[25457]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:03.816 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:03 smithi019 ceph-mon[29267]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:03.816 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:03 smithi019 ceph-mon[24574]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:04.277 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:23:04 smithi172 systemd[1]: Starting Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:23:04.483 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:23:04.531 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:23:04 smithi172 podman[40565]: 2024-02-07 12:23:04.179696184 +0000 UTC m=+0.020267096 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-02-07T12:23:04.531 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:23:04 smithi172 podman[40565]: 2024-02-07 12:23:04.301158219 +0000 UTC m=+0.141729165 container create 244f779cb86f73c1299edc3e526cd98a829eb895cd0022d1104459f3b6139037 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:23:04.531 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:23:04 smithi172 podman[40565]: 2024-02-07 12:23:04.529363167 +0000 UTC m=+0.369934085 container init 244f779cb86f73c1299edc3e526cd98a829eb895cd0022d1104459f3b6139037 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:23:04.733 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:04 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1935547023' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-02-07T12:23:04.733 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:04 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1935547023' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-02-07T12:23:04.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:04 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1935547023' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-02-07T12:23:04.870 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:23:04 smithi172 podman[40565]: 2024-02-07 12:23:04.532066476 +0000 UTC m=+0.372637384 container start 244f779cb86f73c1299edc3e526cd98a829eb895cd0022d1104459f3b6139037 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:23:04.870 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:23:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:23:04.555Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-02-07T12:23:04.870 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:23:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:23:04.555Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-02-07T12:23:04.870 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:23:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:23:04.555Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-02-07T12:23:04.870 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:23:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:23:04.555Z caller=main.go:518 level=info host_details="(Linux 5.14.0-418.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Mon Feb 5 23:17:48 UTC 2024 x86_64 smithi172 (none))" 2024-02-07T12:23:04.870 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:23:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:23:04.555Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-02-07T12:23:04.870 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:23:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:23:04.555Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-02-07T12:23:04.870 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:23:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:23:04.556Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-02-07T12:23:04.870 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:23:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:23:04.557Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-02-07T12:23:04.870 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:23:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:23:04.559Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-02-07T12:23:04.870 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:23:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:23:04.560Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-02-07T12:23:04.871 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:23:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:23:04.560Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.603µs 2024-02-07T12:23:04.871 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:23:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:23:04.560Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-02-07T12:23:04.871 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:23:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:23:04.560Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-02-07T12:23:04.871 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:23:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:23:04.560Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=21.226µs wal_replay_duration=124.865µs total_replay_duration=160.858µs 2024-02-07T12:23:04.871 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:23:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:23:04.561Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-02-07T12:23:04.871 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:23:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:23:04.561Z caller=main.go:947 level=info msg="TSDB started" 2024-02-07T12:23:04.871 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:23:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:23:04.561Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-02-07T12:23:04.871 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:23:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:23:04.576Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.150307ms db_storage=652ns remote_storage=1.139µs web_handler=336ns query_engine=742ns scrape=161.168µs scrape_sd=16.203µs notify=770ns notify_sd=1.315µs rules=14.777899ms 2024-02-07T12:23:04.871 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:23:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:23:04.576Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-02-07T12:23:04.871 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:23:04 smithi172 bash[40565]: 244f779cb86f73c1299edc3e526cd98a829eb895cd0022d1104459f3b6139037 2024-02-07T12:23:04.871 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:23:04 smithi172 systemd[1]: Started Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:23:05.164 INFO:teuthology.orchestra.run.smithi019.stdout:{"epoch":20,"active_gid":14433,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6800","nonce":2683363320},{"type":"v1","addr":"172.21.15.19:6801","nonce":2683363320}]},"active_addr":"172.21.15.19:6801/2683363320","active_change":"2024-02-07T12:22:45.629745+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":24329,"name":"x","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.19:8443/","prometheus":"http://172.21.15.19:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":59,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.19:0","nonce":1464010310}]},{"addrvec":[{"type":"v2","addr":"172.21.15.19:0","nonce":2799591954}]},{"addrvec":[{"type":"v2","addr":"172.21.15.19:0","nonce":3792014269}]},{"addrvec":[{"type":"v2","addr":"172.21.15.19:0","nonce":479908890}]}]}} 2024-02-07T12:23:05.170 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-02-07T12:23:05.170 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-02-07T12:23:05.171 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph osd dump --format=json 2024-02-07T12:23:05.399 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/mon.a/config 2024-02-07T12:23:05.745 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:05 smithi019 ceph-mon[29267]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:05.745 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:05 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:05.745 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:05 smithi019 ceph-mon[24574]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:05.745 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:05 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:05.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:05 smithi172 ceph-mon[25457]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:05.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:05 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:06.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:06 smithi019 ceph-mon[24574]: Deploying daemon node-exporter.a on smithi019 2024-02-07T12:23:06.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:06 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:06.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:06 smithi019 ceph-mon[29267]: Deploying daemon node-exporter.a on smithi019 2024-02-07T12:23:06.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:06 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:06.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:06 smithi172 ceph-mon[25457]: Deploying daemon node-exporter.a on smithi019 2024-02-07T12:23:06.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:06 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:07.262 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:23:07.263 INFO:teuthology.orchestra.run.smithi019.stdout:{"epoch":59,"fsid":"ced2783c-c5b2-11ee-95b6-87774f69a715","created":"2024-02-07T12:17:38.251043+0000","modified":"2024-02-07T12:22:45.629371+0000","last_up_change":"2024-02-07T12:22:30.216345+0000","last_in_change":"2024-02-07T12:22:13.126065+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-02-07T12:20:34.945074+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"3ddfcd81-94a5-46f7-812e-c87f08345aef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6802","nonce":4112372591},{"type":"v1","addr":"172.21.15.19:6803","nonce":4112372591}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6804","nonce":4112372591},{"type":"v1","addr":"172.21.15.19:6805","nonce":4112372591}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6808","nonce":4112372591},{"type":"v1","addr":"172.21.15.19:6809","nonce":4112372591}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6806","nonce":4112372591},{"type":"v1","addr":"172.21.15.19:6807","nonce":4112372591}]},"public_addr":"172.21.15.19:6803/4112372591","cluster_addr":"172.21.15.19:6805/4112372591","heartbeat_back_addr":"172.21.15.19:6809/4112372591","heartbeat_front_addr":"172.21.15.19:6807/4112372591","state":["exists","up"]},{"osd":1,"uuid":"c5266822-bc44-4ede-90a8-bc3f880c1ea8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6810","nonce":3625191644},{"type":"v1","addr":"172.21.15.19:6811","nonce":3625191644}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6812","nonce":3625191644},{"type":"v1","addr":"172.21.15.19:6813","nonce":3625191644}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6816","nonce":3625191644},{"type":"v1","addr":"172.21.15.19:6817","nonce":3625191644}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6814","nonce":3625191644},{"type":"v1","addr":"172.21.15.19:6815","nonce":3625191644}]},"public_addr":"172.21.15.19:6811/3625191644","cluster_addr":"172.21.15.19:6813/3625191644","heartbeat_back_addr":"172.21.15.19:6817/3625191644","heartbeat_front_addr":"172.21.15.19:6815/3625191644","state":["exists","up"]},{"osd":2,"uuid":"1df65fd6-bed8-4ada-ba4c-7cde82b45a31","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6818","nonce":4106092366},{"type":"v1","addr":"172.21.15.19:6819","nonce":4106092366}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6820","nonce":4106092366},{"type":"v1","addr":"172.21.15.19:6821","nonce":4106092366}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6824","nonce":4106092366},{"type":"v1","addr":"172.21.15.19:6825","nonce":4106092366}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6822","nonce":4106092366},{"type":"v1","addr":"172.21.15.19:6823","nonce":4106092366}]},"public_addr":"172.21.15.19:6819/4106092366","cluster_addr":"172.21.15.19:6821/4106092366","heartbeat_back_addr":"172.21.15.19:6825/4106092366","heartbeat_front_addr":"172.21.15.19:6823/4106092366","state":["exists","up"]},{"osd":3,"uuid":"749f4475-2dbd-4166-b49b-3eaf63e1f7bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6826","nonce":2871829035},{"type":"v1","addr":"172.21.15.19:6827","nonce":2871829035}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6828","nonce":2871829035},{"type":"v1","addr":"172.21.15.19:6829","nonce":2871829035}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6832","nonce":2871829035},{"type":"v1","addr":"172.21.15.19:6833","nonce":2871829035}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6830","nonce":2871829035},{"type":"v1","addr":"172.21.15.19:6831","nonce":2871829035}]},"public_addr":"172.21.15.19:6827/2871829035","cluster_addr":"172.21.15.19:6829/2871829035","heartbeat_back_addr":"172.21.15.19:6833/2871829035","heartbeat_front_addr":"172.21.15.19:6831/2871829035","state":["exists","up"]},{"osd":4,"uuid":"7b6de577-7c35-4cc0-bf7e-cddbffbaad2b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6800","nonce":2804555213},{"type":"v1","addr":"172.21.15.172:6801","nonce":2804555213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6802","nonce":2804555213},{"type":"v1","addr":"172.21.15.172:6803","nonce":2804555213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6806","nonce":2804555213},{"type":"v1","addr":"172.21.15.172:6807","nonce":2804555213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6804","nonce":2804555213},{"type":"v1","addr":"172.21.15.172:6805","nonce":2804555213}]},"public_addr":"172.21.15.172:6801/2804555213","cluster_addr":"172.21.15.172:6803/2804555213","heartbeat_back_addr":"172.21.15.172:6807/2804555213","heartbeat_front_addr":"172.21.15.172:6805/2804555213","state":["exists","up"]},{"osd":5,"uuid":"efa49cf9-057d-4a6c-b3b8-368b9497f6ff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6808","nonce":3717679655},{"type":"v1","addr":"172.21.15.172:6809","nonce":3717679655}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6810","nonce":3717679655},{"type":"v1","addr":"172.21.15.172:6811","nonce":3717679655}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6814","nonce":3717679655},{"type":"v1","addr":"172.21.15.172:6815","nonce":3717679655}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6812","nonce":3717679655},{"type":"v1","addr":"172.21.15.172:6813","nonce":3717679655}]},"public_addr":"172.21.15.172:6809/3717679655","cluster_addr":"172.21.15.172:6811/3717679655","heartbeat_back_addr":"172.21.15.172:6815/3717679655","heartbeat_front_addr":"172.21.15.172:6813/3717679655","state":["exists","up"]},{"osd":6,"uuid":"fe43b6a2-9866-4da0-a167-f4ba5564cdbc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6816","nonce":92131068},{"type":"v1","addr":"172.21.15.172:6817","nonce":92131068}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6818","nonce":92131068},{"type":"v1","addr":"172.21.15.172:6819","nonce":92131068}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6822","nonce":92131068},{"type":"v1","addr":"172.21.15.172:6823","nonce":92131068}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6820","nonce":92131068},{"type":"v1","addr":"172.21.15.172:6821","nonce":92131068}]},"public_addr":"172.21.15.172:6817/92131068","cluster_addr":"172.21.15.172:6819/92131068","heartbeat_back_addr":"172.21.15.172:6823/92131068","heartbeat_front_addr":"172.21.15.172:6821/92131068","state":["exists","up"]},{"osd":7,"uuid":"c29a4703-29ad-4c37-a0cb-547860de3480","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6824","nonce":1154934187},{"type":"v1","addr":"172.21.15.172:6825","nonce":1154934187}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6826","nonce":1154934187},{"type":"v1","addr":"172.21.15.172:6827","nonce":1154934187}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6830","nonce":1154934187},{"type":"v1","addr":"172.21.15.172:6831","nonce":1154934187}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6828","nonce":1154934187},{"type":"v1","addr":"172.21.15.172:6829","nonce":1154934187}]},"public_addr":"172.21.15.172:6825/1154934187","cluster_addr":"172.21.15.172:6827/1154934187","heartbeat_back_addr":"172.21.15.172:6831/1154934187","heartbeat_front_addr":"172.21.15.172:6829/1154934187","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:19:46.348476+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:20:09.908222+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:20:33.189734+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:20:55.901187+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:21:17.030966+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:21:40.885450+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:22:04.445626+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:22:28.526412+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.19:0/3548359334":"2024-02-08T12:22:45.629339+0000","172.21.15.19:0/3958473332":"2024-02-08T12:22:45.629339+0000","172.21.15.19:0/840882240":"2024-02-08T12:22:45.629339+0000","172.21.15.19:0/2628806613":"2024-02-08T12:18:27.833418+0000","172.21.15.19:6801/4244076805":"2024-02-08T12:22:45.629339+0000","172.21.15.19:6800/4244076805":"2024-02-08T12:22:45.629339+0000","172.21.15.19:6800/2471401560":"2024-02-08T12:18:27.833418+0000","172.21.15.19:0/784284954":"2024-02-08T12:18:27.833418+0000","172.21.15.19:0/412909863":"2024-02-08T12:22:45.629339+0000","172.21.15.19:6800/2571038478":"2024-02-08T12:18:05.905616+0000","172.21.15.19:6801/2571038478":"2024-02-08T12:18:05.905616+0000","172.21.15.19:6801/2471401560":"2024-02-08T12:18:27.833418+0000","172.21.15.19:0/1231318565":"2024-02-08T12:18:05.905616+0000","172.21.15.19:0/2080165887":"2024-02-08T12:18:27.833418+0000","172.21.15.19:0/1819641709":"2024-02-08T12:18:05.905616+0000","172.21.15.19:0/4109166938":"2024-02-08T12:18:05.905616+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-02-07T12:23:07.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:07 smithi019 ceph-mon[24574]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:07.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:07 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2711207873' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-02-07T12:23:07.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:07 smithi019 ceph-mon[29267]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:07.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:07 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2711207873' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-02-07T12:23:07.845 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:07 smithi019 bash[44118]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-02-07T12:23:07.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:07 smithi172 ceph-mon[25457]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:07.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:07 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2711207873' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-02-07T12:23:07.912 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-02-07T12:23:07.912 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph osd dump --format=json 2024-02-07T12:23:08.132 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/mon.a/config 2024-02-07T12:23:08.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:08 smithi019 ceph-mon[24574]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:08.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:08 smithi019 ceph-mon[29267]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:09.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:08 smithi172 ceph-mon[25457]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:10.006 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:09 smithi019 bash[44118]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-02-07T12:23:10.006 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:09 smithi019 bash[44118]: Writing manifest to image destination 2024-02-07T12:23:10.454 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:23:10.454 INFO:teuthology.orchestra.run.smithi019.stdout:{"epoch":59,"fsid":"ced2783c-c5b2-11ee-95b6-87774f69a715","created":"2024-02-07T12:17:38.251043+0000","modified":"2024-02-07T12:22:45.629371+0000","last_up_change":"2024-02-07T12:22:30.216345+0000","last_in_change":"2024-02-07T12:22:13.126065+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-02-07T12:20:34.945074+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"3ddfcd81-94a5-46f7-812e-c87f08345aef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6802","nonce":4112372591},{"type":"v1","addr":"172.21.15.19:6803","nonce":4112372591}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6804","nonce":4112372591},{"type":"v1","addr":"172.21.15.19:6805","nonce":4112372591}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6808","nonce":4112372591},{"type":"v1","addr":"172.21.15.19:6809","nonce":4112372591}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6806","nonce":4112372591},{"type":"v1","addr":"172.21.15.19:6807","nonce":4112372591}]},"public_addr":"172.21.15.19:6803/4112372591","cluster_addr":"172.21.15.19:6805/4112372591","heartbeat_back_addr":"172.21.15.19:6809/4112372591","heartbeat_front_addr":"172.21.15.19:6807/4112372591","state":["exists","up"]},{"osd":1,"uuid":"c5266822-bc44-4ede-90a8-bc3f880c1ea8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6810","nonce":3625191644},{"type":"v1","addr":"172.21.15.19:6811","nonce":3625191644}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6812","nonce":3625191644},{"type":"v1","addr":"172.21.15.19:6813","nonce":3625191644}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6816","nonce":3625191644},{"type":"v1","addr":"172.21.15.19:6817","nonce":3625191644}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6814","nonce":3625191644},{"type":"v1","addr":"172.21.15.19:6815","nonce":3625191644}]},"public_addr":"172.21.15.19:6811/3625191644","cluster_addr":"172.21.15.19:6813/3625191644","heartbeat_back_addr":"172.21.15.19:6817/3625191644","heartbeat_front_addr":"172.21.15.19:6815/3625191644","state":["exists","up"]},{"osd":2,"uuid":"1df65fd6-bed8-4ada-ba4c-7cde82b45a31","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6818","nonce":4106092366},{"type":"v1","addr":"172.21.15.19:6819","nonce":4106092366}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6820","nonce":4106092366},{"type":"v1","addr":"172.21.15.19:6821","nonce":4106092366}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6824","nonce":4106092366},{"type":"v1","addr":"172.21.15.19:6825","nonce":4106092366}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6822","nonce":4106092366},{"type":"v1","addr":"172.21.15.19:6823","nonce":4106092366}]},"public_addr":"172.21.15.19:6819/4106092366","cluster_addr":"172.21.15.19:6821/4106092366","heartbeat_back_addr":"172.21.15.19:6825/4106092366","heartbeat_front_addr":"172.21.15.19:6823/4106092366","state":["exists","up"]},{"osd":3,"uuid":"749f4475-2dbd-4166-b49b-3eaf63e1f7bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6826","nonce":2871829035},{"type":"v1","addr":"172.21.15.19:6827","nonce":2871829035}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6828","nonce":2871829035},{"type":"v1","addr":"172.21.15.19:6829","nonce":2871829035}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6832","nonce":2871829035},{"type":"v1","addr":"172.21.15.19:6833","nonce":2871829035}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.19:6830","nonce":2871829035},{"type":"v1","addr":"172.21.15.19:6831","nonce":2871829035}]},"public_addr":"172.21.15.19:6827/2871829035","cluster_addr":"172.21.15.19:6829/2871829035","heartbeat_back_addr":"172.21.15.19:6833/2871829035","heartbeat_front_addr":"172.21.15.19:6831/2871829035","state":["exists","up"]},{"osd":4,"uuid":"7b6de577-7c35-4cc0-bf7e-cddbffbaad2b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6800","nonce":2804555213},{"type":"v1","addr":"172.21.15.172:6801","nonce":2804555213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6802","nonce":2804555213},{"type":"v1","addr":"172.21.15.172:6803","nonce":2804555213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6806","nonce":2804555213},{"type":"v1","addr":"172.21.15.172:6807","nonce":2804555213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6804","nonce":2804555213},{"type":"v1","addr":"172.21.15.172:6805","nonce":2804555213}]},"public_addr":"172.21.15.172:6801/2804555213","cluster_addr":"172.21.15.172:6803/2804555213","heartbeat_back_addr":"172.21.15.172:6807/2804555213","heartbeat_front_addr":"172.21.15.172:6805/2804555213","state":["exists","up"]},{"osd":5,"uuid":"efa49cf9-057d-4a6c-b3b8-368b9497f6ff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6808","nonce":3717679655},{"type":"v1","addr":"172.21.15.172:6809","nonce":3717679655}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6810","nonce":3717679655},{"type":"v1","addr":"172.21.15.172:6811","nonce":3717679655}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6814","nonce":3717679655},{"type":"v1","addr":"172.21.15.172:6815","nonce":3717679655}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6812","nonce":3717679655},{"type":"v1","addr":"172.21.15.172:6813","nonce":3717679655}]},"public_addr":"172.21.15.172:6809/3717679655","cluster_addr":"172.21.15.172:6811/3717679655","heartbeat_back_addr":"172.21.15.172:6815/3717679655","heartbeat_front_addr":"172.21.15.172:6813/3717679655","state":["exists","up"]},{"osd":6,"uuid":"fe43b6a2-9866-4da0-a167-f4ba5564cdbc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6816","nonce":92131068},{"type":"v1","addr":"172.21.15.172:6817","nonce":92131068}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6818","nonce":92131068},{"type":"v1","addr":"172.21.15.172:6819","nonce":92131068}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6822","nonce":92131068},{"type":"v1","addr":"172.21.15.172:6823","nonce":92131068}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6820","nonce":92131068},{"type":"v1","addr":"172.21.15.172:6821","nonce":92131068}]},"public_addr":"172.21.15.172:6817/92131068","cluster_addr":"172.21.15.172:6819/92131068","heartbeat_back_addr":"172.21.15.172:6823/92131068","heartbeat_front_addr":"172.21.15.172:6821/92131068","state":["exists","up"]},{"osd":7,"uuid":"c29a4703-29ad-4c37-a0cb-547860de3480","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6824","nonce":1154934187},{"type":"v1","addr":"172.21.15.172:6825","nonce":1154934187}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6826","nonce":1154934187},{"type":"v1","addr":"172.21.15.172:6827","nonce":1154934187}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6830","nonce":1154934187},{"type":"v1","addr":"172.21.15.172:6831","nonce":1154934187}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6828","nonce":1154934187},{"type":"v1","addr":"172.21.15.172:6829","nonce":1154934187}]},"public_addr":"172.21.15.172:6825/1154934187","cluster_addr":"172.21.15.172:6827/1154934187","heartbeat_back_addr":"172.21.15.172:6831/1154934187","heartbeat_front_addr":"172.21.15.172:6829/1154934187","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:19:46.348476+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:20:09.908222+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:20:33.189734+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:20:55.901187+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:21:17.030966+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:21:40.885450+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:22:04.445626+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:22:28.526412+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.19:0/3548359334":"2024-02-08T12:22:45.629339+0000","172.21.15.19:0/3958473332":"2024-02-08T12:22:45.629339+0000","172.21.15.19:0/840882240":"2024-02-08T12:22:45.629339+0000","172.21.15.19:0/2628806613":"2024-02-08T12:18:27.833418+0000","172.21.15.19:6801/4244076805":"2024-02-08T12:22:45.629339+0000","172.21.15.19:6800/4244076805":"2024-02-08T12:22:45.629339+0000","172.21.15.19:6800/2471401560":"2024-02-08T12:18:27.833418+0000","172.21.15.19:0/784284954":"2024-02-08T12:18:27.833418+0000","172.21.15.19:0/412909863":"2024-02-08T12:22:45.629339+0000","172.21.15.19:6800/2571038478":"2024-02-08T12:18:05.905616+0000","172.21.15.19:6801/2571038478":"2024-02-08T12:18:05.905616+0000","172.21.15.19:6801/2471401560":"2024-02-08T12:18:27.833418+0000","172.21.15.19:0/1231318565":"2024-02-08T12:18:05.905616+0000","172.21.15.19:0/2080165887":"2024-02-08T12:18:27.833418+0000","172.21.15.19:0/1819641709":"2024-02-08T12:18:05.905616+0000","172.21.15.19:0/4109166938":"2024-02-08T12:18:05.905616+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-02-07T12:23:10.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:10 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2087736130' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-02-07T12:23:10.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:10 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2087736130' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-02-07T12:23:10.595 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:10 smithi019 podman[44118]: 2024-02-07 12:23:10.324864919 +0000 UTC m=+3.462272424 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-02-07T12:23:10.595 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:10 smithi019 podman[44118]: 2024-02-07 12:23:10.406632561 +0000 UTC m=+3.544040061 container create de9599dbfeff9861b6e6afb015e173ba658c5219e6c13d1aec84f6bd7272dbb7 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-02-07T12:23:10.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:10 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2087736130' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-02-07T12:23:11.181 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph tell osd.0 flush_pg_stats 2024-02-07T12:23:11.182 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph tell osd.1 flush_pg_stats 2024-02-07T12:23:11.182 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph tell osd.2 flush_pg_stats 2024-02-07T12:23:11.182 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph tell osd.3 flush_pg_stats 2024-02-07T12:23:11.183 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph tell osd.4 flush_pg_stats 2024-02-07T12:23:11.183 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph tell osd.5 flush_pg_stats 2024-02-07T12:23:11.184 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph tell osd.6 flush_pg_stats 2024-02-07T12:23:11.184 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph tell osd.7 flush_pg_stats 2024-02-07T12:23:11.407 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 podman[44118]: 2024-02-07 12:23:11.338985824 +0000 UTC m=+4.476393343 container init de9599dbfeff9861b6e6afb015e173ba658c5219e6c13d1aec84f6bd7272dbb7 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-02-07T12:23:11.408 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 podman[44118]: 2024-02-07 12:23:11.344599341 +0000 UTC m=+4.482006851 container start de9599dbfeff9861b6e6afb015e173ba658c5219e6c13d1aec84f6bd7272dbb7 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-02-07T12:23:11.408 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.399Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-02-07T12:23:11.408 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.399Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-02-07T12:23:11.408 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.400Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-02-07T12:23:11.408 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.400Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-02-07T12:23:11.408 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-02-07T12:23:11.408 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=arp 2024-02-07T12:23:11.408 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=bcache 2024-02-07T12:23:11.408 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=bonding 2024-02-07T12:23:11.409 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=btrfs 2024-02-07T12:23:11.409 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=conntrack 2024-02-07T12:23:11.409 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=cpu 2024-02-07T12:23:11.409 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-02-07T12:23:11.409 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=diskstats 2024-02-07T12:23:11.409 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=dmi 2024-02-07T12:23:11.409 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=edac 2024-02-07T12:23:11.410 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=entropy 2024-02-07T12:23:11.410 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-02-07T12:23:11.410 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=filefd 2024-02-07T12:23:11.410 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=filesystem 2024-02-07T12:23:11.410 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=hwmon 2024-02-07T12:23:11.410 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=infiniband 2024-02-07T12:23:11.410 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=ipvs 2024-02-07T12:23:11.410 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=loadavg 2024-02-07T12:23:11.410 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=mdadm 2024-02-07T12:23:11.411 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=meminfo 2024-02-07T12:23:11.411 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=netclass 2024-02-07T12:23:11.411 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=netdev 2024-02-07T12:23:11.411 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=netstat 2024-02-07T12:23:11.411 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=nfs 2024-02-07T12:23:11.411 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=nfsd 2024-02-07T12:23:11.411 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=nvme 2024-02-07T12:23:11.411 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=os 2024-02-07T12:23:11.411 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-02-07T12:23:11.411 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=pressure 2024-02-07T12:23:11.411 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=rapl 2024-02-07T12:23:11.411 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=schedstat 2024-02-07T12:23:11.412 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=sockstat 2024-02-07T12:23:11.412 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=softnet 2024-02-07T12:23:11.412 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=stat 2024-02-07T12:23:11.412 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=tapestats 2024-02-07T12:23:11.412 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=textfile 2024-02-07T12:23:11.412 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-02-07T12:23:11.412 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=time 2024-02-07T12:23:11.412 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-02-07T12:23:11.412 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=uname 2024-02-07T12:23:11.412 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=vmstat 2024-02-07T12:23:11.413 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=xfs 2024-02-07T12:23:11.413 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.401Z caller=node_exporter.go:115 level=info collector=zfs 2024-02-07T12:23:11.413 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.402Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-02-07T12:23:11.413 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[44345]: ts=2024-02-07T12:23:11.402Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-02-07T12:23:11.616 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/mon.a/config 2024-02-07T12:23:11.645 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/mon.a/config 2024-02-07T12:23:11.646 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/mon.a/config 2024-02-07T12:23:11.647 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/mon.a/config 2024-02-07T12:23:11.649 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/mon.a/config 2024-02-07T12:23:11.651 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/mon.a/config 2024-02-07T12:23:11.651 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/mon.a/config 2024-02-07T12:23:11.670 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/mon.a/config 2024-02-07T12:23:11.737 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-mon[29267]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:11.738 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:11 smithi019 ceph-mon[24574]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:11.738 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 bash[44118]: de9599dbfeff9861b6e6afb015e173ba658c5219e6c13d1aec84f6bd7272dbb7 2024-02-07T12:23:11.738 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:23:11 smithi019 systemd[1]: Started Ceph node-exporter.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:23:11.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:11 smithi172 ceph-mon[25457]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:12.577 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:12 smithi172 bash[40806]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-02-07T12:23:12.789 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:12 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:12.789 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:12 smithi019 ceph-mon[29267]: Deploying daemon node-exporter.b on smithi172 2024-02-07T12:23:12.789 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:12 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:12.789 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:12 smithi019 ceph-mon[24574]: Deploying daemon node-exporter.b on smithi172 2024-02-07T12:23:12.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:12 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:12.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:12 smithi172 ceph-mon[25457]: Deploying daemon node-exporter.b on smithi172 2024-02-07T12:23:12.869 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:12 smithi172 bash[40806]: Getting image source signatures 2024-02-07T12:23:12.869 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:12 smithi172 bash[40806]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-02-07T12:23:12.870 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:12 smithi172 bash[40806]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-02-07T12:23:12.870 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:12 smithi172 bash[40806]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-02-07T12:23:13.828 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:13 smithi019 ceph-mon[29267]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:13.829 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:13 smithi019 ceph-mon[24574]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:13.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:13 smithi172 ceph-mon[25457]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:14.869 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:14 smithi172 bash[40806]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-02-07T12:23:14.869 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:14 smithi172 bash[40806]: Writing manifest to image destination 2024-02-07T12:23:15.302 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:14 smithi172 podman[40806]: 2024-02-07 12:23:14.932468536 +0000 UTC m=+2.579284807 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-02-07T12:23:15.302 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 podman[40806]: 2024-02-07 12:23:15.022734992 +0000 UTC m=+2.669551323 container create 30f1dfd23ff27186b789d921d270bee835dfa0856df3bb586d7db1513482ea82 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-02-07T12:23:15.302 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 podman[40806]: 2024-02-07 12:23:15.199455948 +0000 UTC m=+2.846272219 container init 30f1dfd23ff27186b789d921d270bee835dfa0856df3bb586d7db1513482ea82 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-02-07T12:23:15.302 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 podman[40806]: 2024-02-07 12:23:15.202169942 +0000 UTC m=+2.848986219 container start 30f1dfd23ff27186b789d921d270bee835dfa0856df3bb586d7db1513482ea82 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-02-07T12:23:15.302 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.205Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-02-07T12:23:15.302 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.205Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-02-07T12:23:15.302 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.206Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-02-07T12:23:15.302 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.206Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-02-07T12:23:15.303 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.206Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-02-07T12:23:15.303 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.206Z caller=node_exporter.go:115 level=info collector=arp 2024-02-07T12:23:15.303 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.206Z caller=node_exporter.go:115 level=info collector=bcache 2024-02-07T12:23:15.303 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.206Z caller=node_exporter.go:115 level=info collector=bonding 2024-02-07T12:23:15.303 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.206Z caller=node_exporter.go:115 level=info collector=btrfs 2024-02-07T12:23:15.303 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.206Z caller=node_exporter.go:115 level=info collector=conntrack 2024-02-07T12:23:15.303 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.206Z caller=node_exporter.go:115 level=info collector=cpu 2024-02-07T12:23:15.303 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-02-07T12:23:15.303 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=diskstats 2024-02-07T12:23:15.303 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=dmi 2024-02-07T12:23:15.303 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=edac 2024-02-07T12:23:15.303 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=entropy 2024-02-07T12:23:15.303 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-02-07T12:23:15.303 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=filefd 2024-02-07T12:23:15.304 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=filesystem 2024-02-07T12:23:15.304 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=hwmon 2024-02-07T12:23:15.304 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=infiniband 2024-02-07T12:23:15.304 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=ipvs 2024-02-07T12:23:15.304 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=loadavg 2024-02-07T12:23:15.304 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=mdadm 2024-02-07T12:23:15.304 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=meminfo 2024-02-07T12:23:15.304 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=netclass 2024-02-07T12:23:15.304 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=netdev 2024-02-07T12:23:15.304 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=netstat 2024-02-07T12:23:15.304 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=nfs 2024-02-07T12:23:15.304 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=nfsd 2024-02-07T12:23:15.304 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=nvme 2024-02-07T12:23:15.304 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=os 2024-02-07T12:23:15.305 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-02-07T12:23:15.305 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=pressure 2024-02-07T12:23:15.305 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=rapl 2024-02-07T12:23:15.305 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=schedstat 2024-02-07T12:23:15.305 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=sockstat 2024-02-07T12:23:15.305 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=softnet 2024-02-07T12:23:15.305 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=stat 2024-02-07T12:23:15.305 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=tapestats 2024-02-07T12:23:15.305 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=textfile 2024-02-07T12:23:15.305 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-02-07T12:23:15.305 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=time 2024-02-07T12:23:15.305 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-02-07T12:23:15.305 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=uname 2024-02-07T12:23:15.305 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=vmstat 2024-02-07T12:23:15.306 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=xfs 2024-02-07T12:23:15.306 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:115 level=info collector=zfs 2024-02-07T12:23:15.306 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-02-07T12:23:15.306 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[40857]: ts=2024-02-07T12:23:15.207Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-02-07T12:23:15.306 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 bash[40806]: 30f1dfd23ff27186b789d921d270bee835dfa0856df3bb586d7db1513482ea82 2024-02-07T12:23:15.581 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:23:15 smithi172 systemd[1]: Started Ceph node-exporter.b for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:23:15.782 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:15 smithi019 ceph-mon[24574]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:15.782 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:15 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:15.782 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:15 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:23:15.782 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:15 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:23:15.782 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:15 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:23:15.783 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:15 smithi019 ceph-mon[29267]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:15.783 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:15 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:15.783 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:15 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:23:15.783 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:15 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:23:15.783 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:15 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:23:15.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-mon[25457]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:15.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:15.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:23:15.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:23:15.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:15 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:23:17.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:17 smithi019 ceph-mon[24574]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:17.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:17 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:17.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:17 smithi019 ceph-mon[29267]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:17.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:17 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:17.784 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:17 smithi172 ceph-mon[25457]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:17.784 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:17 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:18.037 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:23:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:23:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:23:18.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:23:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:23:18] "GET /metrics HTTP/1.1" 200 191500 "" "Prometheus/2.33.4" 2024-02-07T12:23:19.313 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:19 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:19.313 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:19 smithi019 ceph-mon[24574]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:19.314 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:19 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:19.314 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:19 smithi019 ceph-mon[29267]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:19.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:19 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:19.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:19 smithi172 ceph-mon[25457]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:21.432 INFO:teuthology.orchestra.run.smithi019.stdout:231928233995 2024-02-07T12:23:21.432 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph osd last-stat-seq osd.7 2024-02-07T12:23:21.673 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:21 smithi019 ceph-mon[24574]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:21.674 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:21 smithi019 ceph-mon[29267]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:21.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:21 smithi172 ceph-mon[25457]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:22.811 INFO:teuthology.orchestra.run.smithi019.stdout:55834574888 2024-02-07T12:23:22.812 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph osd last-stat-seq osd.1 2024-02-07T12:23:23.728 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:23 smithi019 ceph-mon[29267]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:23.728 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:23 smithi019 ceph-mon[24574]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:23.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:23 smithi172 ceph-mon[25457]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:24.854 INFO:teuthology.orchestra.run.smithi019.stdout:107374182431 2024-02-07T12:23:24.854 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph osd last-stat-seq osd.3 2024-02-07T12:23:25.679 INFO:teuthology.orchestra.run.smithi019.stdout:197568495631 2024-02-07T12:23:25.679 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph osd last-stat-seq osd.6 2024-02-07T12:23:25.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:25 smithi019 ceph-mon[29267]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:25.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:25 smithi019 ceph-mon[24574]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:25.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:25 smithi172 ceph-mon[25457]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:26.241 INFO:teuthology.orchestra.run.smithi019.stdout:77309411364 2024-02-07T12:23:26.241 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph osd last-stat-seq osd.2 2024-02-07T12:23:26.641 INFO:teuthology.orchestra.run.smithi019.stdout:34359738414 2024-02-07T12:23:26.642 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph osd last-stat-seq osd.0 2024-02-07T12:23:27.016 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/mon.a/config 2024-02-07T12:23:27.415 INFO:teuthology.orchestra.run.smithi019.stdout:128849018907 2024-02-07T12:23:27.416 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph osd last-stat-seq osd.4 2024-02-07T12:23:27.446 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/mon.a/config 2024-02-07T12:23:27.453 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/mon.a/config 2024-02-07T12:23:27.454 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/mon.a/config 2024-02-07T12:23:27.679 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:27 smithi019 ceph-mon[24574]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:27.679 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:27 smithi019 ceph-mon[29267]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:27.690 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/mon.a/config 2024-02-07T12:23:27.699 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/mon.a/config 2024-02-07T12:23:27.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:27 smithi172 ceph-mon[25457]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:28.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:23:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:23:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:23:28.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:23:28 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:23:28] "GET /metrics HTTP/1.1" 200 191500 "" "Prometheus/2.33.4" 2024-02-07T12:23:28.957 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:28 smithi019 ceph-mon[29267]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:28.957 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:28 smithi019 ceph-mon[24574]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:29.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:28 smithi172 ceph-mon[25457]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:29.620 INFO:teuthology.orchestra.run.smithi019.stdout:163208757271 2024-02-07T12:23:29.620 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph osd last-stat-seq osd.5 2024-02-07T12:23:31.225 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/mon.a/config 2024-02-07T12:23:31.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:31 smithi019 ceph-mon[24574]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:31.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:31 smithi019 ceph-mon[29267]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:31.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:31 smithi172 ceph-mon[25457]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:33.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:33 smithi019 ceph-mon[24574]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:33.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:33 smithi019 ceph-mon[29267]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:33.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:33 smithi172 ceph-mon[25457]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:35.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:35 smithi019 ceph-mon[24574]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:35.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:35 smithi019 ceph-mon[29267]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:35.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:35 smithi172 ceph-mon[25457]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:36.245 INFO:teuthology.orchestra.run.smithi019.stdout:107374182433 2024-02-07T12:23:36.773 INFO:teuthology.orchestra.run.smithi019.stdout:197568495634 2024-02-07T12:23:36.774 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:36 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1835329992' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-02-07T12:23:36.774 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:36 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1835329992' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-02-07T12:23:36.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:36 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1835329992' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-02-07T12:23:37.640 INFO:teuthology.orchestra.run.smithi019.stdout:77309411366 2024-02-07T12:23:37.641 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:37 smithi019 ceph-mon[29267]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:37.641 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:37 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3595668563' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-02-07T12:23:37.641 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:37 smithi019 ceph-mon[24574]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:37.641 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:37 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3595668563' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-02-07T12:23:37.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:37 smithi172 ceph-mon[25457]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:37.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:37 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3595668563' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-02-07T12:23:38.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:23:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:23:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:23:38.432 INFO:teuthology.orchestra.run.smithi019.stdout:34359738416 2024-02-07T12:23:38.571 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:23:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:23:38] "GET /metrics HTTP/1.1" 200 191489 "" "Prometheus/2.33.4" 2024-02-07T12:23:38.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:38 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/202324924' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-02-07T12:23:38.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:38 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2322503620' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-02-07T12:23:38.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:38 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/202324924' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-02-07T12:23:38.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:38 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2322503620' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-02-07T12:23:38.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:38 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/202324924' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-02-07T12:23:38.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:38 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2322503620' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-02-07T12:23:39.599 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182431 got 107374182433 for osd.3 2024-02-07T12:23:39.600 DEBUG:teuthology.parallel:result is None 2024-02-07T12:23:39.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:39 smithi019 ceph-mon[24574]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:39.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:39 smithi019 ceph-mon[29267]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:39.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:39 smithi172 ceph-mon[25457]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:40.700 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/mon.a/config 2024-02-07T12:23:40.729 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495631 got 197568495634 for osd.6 2024-02-07T12:23:40.729 DEBUG:teuthology.parallel:result is None 2024-02-07T12:23:41.347 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738414 got 34359738416 for osd.0 2024-02-07T12:23:41.347 DEBUG:teuthology.parallel:result is None 2024-02-07T12:23:41.728 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411364 got 77309411366 for osd.2 2024-02-07T12:23:41.728 DEBUG:teuthology.parallel:result is None 2024-02-07T12:23:41.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:41 smithi019 ceph-mon[29267]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:41.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:41 smithi019 ceph-mon[24574]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:41.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:41 smithi172 ceph-mon[25457]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:42.079 INFO:teuthology.orchestra.run.smithi019.stdout:128849018910 2024-02-07T12:23:42.840 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:42 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3356702052' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-02-07T12:23:42.840 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:42 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3356702052' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-02-07T12:23:42.842 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018907 got 128849018910 for osd.4 2024-02-07T12:23:42.842 DEBUG:teuthology.parallel:result is None 2024-02-07T12:23:42.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:42 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3356702052' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-02-07T12:23:43.449 INFO:teuthology.orchestra.run.smithi019.stdout:231928234000 2024-02-07T12:23:43.758 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:43 smithi019 ceph-mon[29267]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:43.758 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:43 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/303997879' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-02-07T12:23:43.758 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:43 smithi019 ceph-mon[24574]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:43.758 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:43 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/303997879' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-02-07T12:23:43.872 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:43 smithi172 ceph-mon[25457]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:43.872 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:43 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/303997879' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-02-07T12:23:44.984 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928233995 got 231928234000 for osd.7 2024-02-07T12:23:44.984 DEBUG:teuthology.parallel:result is None 2024-02-07T12:23:45.050 INFO:teuthology.orchestra.run.smithi019.stdout:55834574893 2024-02-07T12:23:45.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:45 smithi019 ceph-mon[24574]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:45.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:45 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/821890313' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-02-07T12:23:45.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:45 smithi019 ceph-mon[29267]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:45.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:45 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/821890313' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-02-07T12:23:45.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:45 smithi172 ceph-mon[25457]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:45.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:45 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/821890313' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-02-07T12:23:46.950 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574888 got 55834574893 for osd.1 2024-02-07T12:23:46.950 DEBUG:teuthology.parallel:result is None 2024-02-07T12:23:47.504 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:47 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:47.504 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:47 smithi019 ceph-mon[29267]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:47.504 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:47 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:47.504 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:47 smithi019 ceph-mon[24574]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:47.504 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:47 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:47.504 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:47 smithi019 ceph-mon[24574]: Deploying daemon alertmanager.a on smithi019 2024-02-07T12:23:47.504 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:23:47.504 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:47 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:23:47.504 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:23:47.504 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:47 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:23:47.638 INFO:teuthology.orchestra.run.smithi019.stdout:163208757274 2024-02-07T12:23:47.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:47 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:47.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:47 smithi019 ceph-mon[29267]: Deploying daemon alertmanager.a on smithi019 2024-02-07T12:23:47.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:23:47.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:47 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:23:47.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:23:47.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:47 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:23:47.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:47 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:47.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:47 smithi172 ceph-mon[25457]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:47.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:47 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:47.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:47 smithi172 ceph-mon[25457]: Deploying daemon alertmanager.a on smithi019 2024-02-07T12:23:47.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:23:47.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:47 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:23:47.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:23:47.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:47 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:23:48.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:23:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:23:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:23:48.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:48 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2467566712' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-02-07T12:23:48.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:23:48 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:23:48] "GET /metrics HTTP/1.1" 200 191492 "" "Prometheus/2.33.4" 2024-02-07T12:23:48.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:48 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2467566712' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-02-07T12:23:48.598 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757271 got 163208757274 for osd.5 2024-02-07T12:23:48.598 DEBUG:teuthology.parallel:result is None 2024-02-07T12:23:48.599 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-02-07T12:23:48.599 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph pg dump --format=json 2024-02-07T12:23:48.852 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/mon.a/config 2024-02-07T12:23:48.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:48 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2467566712' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-02-07T12:23:49.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:49 smithi019 ceph-mon[29267]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:49.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:49 smithi019 ceph-mon[24574]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:49.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:49 smithi172 ceph-mon[25457]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:51.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:51 smithi019 ceph-mon[29267]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:51.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:51 smithi019 ceph-mon[24574]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:51.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:51 smithi172 ceph-mon[25457]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:53.693 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:53 smithi019 ceph-mon[24574]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:53.693 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:53 smithi019 ceph-mon[29267]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:53.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:53 smithi172 ceph-mon[25457]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:54.504 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:23:54.505 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2024-02-07T12:23:55.529 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":36,"stamp":"2024-02-07T12:23:52.506106+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48908,"kb_used_data":6092,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912308,"statfs":{"total":767960285184,"available":767910203392,"internally_reserved":0,"allocated":6238208,"data_stored":3488717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002047"},"pg_stats":[{"pgid":"1.0","version":"59'128","reported_seq":266,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-02-07T12:22:46.591374+0000","last_change":"2024-02-07T12:22:33.685902+0000","last_active":"2024-02-07T12:22:46.591374+0000","last_peered":"2024-02-07T12:22:46.591374+0000","last_clean":"2024-02-07T12:22:46.591374+0000","last_became_active":"2024-02-07T12:22:33.685384+0000","last_became_peered":"2024-02-07T12:22:33.685384+0000","last_unstale":"2024-02-07T12:22:46.591374+0000","last_undegraded":"2024-02-07T12:22:46.591374+0000","last_fullsized":"2024-02-07T12:22:46.591374+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-02-07T12:20:35.685309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-02-07T12:20:35.685309+0000","last_clean_scrub_stamp":"2024-02-07T12:20:35.685309+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-02-08T17:08:04.236560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234002,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6088,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739064,"statfs":{"total":95995035648,"available":95988801536,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Feb 7 12:23:34 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.46100000000000002}]},{"osd":1,"last update":"Wed Feb 7 12:23:34 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.40500000000000003}]},{"osd":2,"last update":"Wed Feb 7 12:23:34 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.54600000000000004}]},{"osd":3,"last update":"Wed Feb 7 12:23:34 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.503}]},{"osd":4,"last update":"Wed Feb 7 12:23:34 2024","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.443}]},{"osd":5,"last update":"Wed Feb 7 12:23:34 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.63900000000000001}]},{"osd":6,"last update":"Wed Feb 7 12:23:34 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.57699999999999996}]}]},{"osd":6,"up_from":46,"seq":197568495638,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6148,"kb_used_data":1084,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739004,"statfs":{"total":95995035648,"available":95988740096,"internally_reserved":0,"allocated":1110016,"data_stored":766504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Feb 7 12:23:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.52200000000000002}]},{"osd":1,"last update":"Wed Feb 7 12:23:09 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.623},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.60099999999999998}]},{"osd":2,"last update":"Wed Feb 7 12:23:09 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.80900000000000005}]},{"osd":3,"last update":"Wed Feb 7 12:23:09 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.79600000000000004},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.74099999999999999}]},{"osd":4,"last update":"Wed Feb 7 12:23:09 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.66800000000000004}]},{"osd":5,"last update":"Wed Feb 7 12:23:09 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.77000000000000002}]},{"osd":7,"last update":"Wed Feb 7 12:23:34 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.75},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.70299999999999996}]}]},{"osd":1,"up_from":13,"seq":55834574895,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6328,"kb_used_data":560,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738824,"statfs":{"total":95995035648,"available":95988555776,"internally_reserved":0,"allocated":573440,"data_stored":237311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Feb 7 12:23:20 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.35099999999999998,"15min":0.33800000000000002},"min":{"1min":0.24099999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.56799999999999995,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.379,"15min":0.35999999999999999},"min":{"1min":0.29699999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.67100000000000004}]},{"osd":2,"last update":"Wed Feb 7 12:23:39 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.39900000000000002,"15min":0.40100000000000002},"min":{"1min":0.26600000000000001,"5min":0.246,"15min":0.246},"max":{"1min":0.59199999999999997,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.41599999999999998,"15min":0.41799999999999998},"min":{"1min":0.29099999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.61799999999999999,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.53700000000000003}]},{"osd":3,"last update":"Wed Feb 7 12:23:01 2024","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.45300000000000001,"15min":0.45200000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.47999999999999998,"15min":0.48299999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.63800000000000001}]},{"osd":4,"last update":"Wed Feb 7 12:23:24 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.57199999999999995,"15min":0.57299999999999995},"min":{"1min":0.44700000000000001,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.72799999999999998,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.40100000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.80000000000000004}]},{"osd":5,"last update":"Wed Feb 7 12:23:48 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.57299999999999995,"15min":0.56699999999999995},"min":{"1min":0.47999999999999998,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.97099999999999997},{"interface":"front","average":{"1min":0.62,"5min":0.60799999999999998,"15min":0.60599999999999998},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.90400000000000003}]},{"osd":6,"last update":"Wed Feb 7 12:23:08 2024","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.91500000000000004},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.79400000000000004}]},{"osd":7,"last update":"Wed Feb 7 12:23:31 2024","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.878},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.93899999999999995}]}]},{"osd":0,"up_from":8,"seq":34359738420,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6920,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738232,"statfs":{"total":95995035648,"available":95987949568,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Feb 7 12:23:18 2024","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.36699999999999999,"15min":0.35999999999999999},"min":{"1min":0.253,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.51600000000000001,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.376,"15min":0.34799999999999998},"min":{"1min":0.32000000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.38500000000000001}]},{"osd":2,"last update":"Wed Feb 7 12:23:38 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.41799999999999998,"15min":0.42499999999999999},"min":{"1min":0.29299999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.53400000000000003,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.436,"15min":0.44},"min":{"1min":0.28100000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.63300000000000001,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.375}]},{"osd":3,"last update":"Wed Feb 7 12:23:01 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.46999999999999997,"15min":0.47199999999999998},"min":{"1min":0.31,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.505},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.45100000000000001,"15min":0.44900000000000001},"min":{"1min":0.33300000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.436}]},{"osd":4,"last update":"Wed Feb 7 12:23:22 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.56899999999999995,"15min":0.57099999999999995},"min":{"1min":0.432,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.75900000000000001,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.59099999999999997,"15min":0.59299999999999997},"min":{"1min":0.437,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.749,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.69299999999999995}]},{"osd":5,"last update":"Wed Feb 7 12:23:46 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.61299999999999999,"15min":0.62},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.79000000000000004,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.59099999999999997,"15min":0.59199999999999997},"min":{"1min":0.45300000000000001,"5min":0.434,"15min":0.434},"max":{"1min":0.751,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.61099999999999999}]},{"osd":6,"last update":"Wed Feb 7 12:23:07 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.66200000000000003}]},{"osd":7,"last update":"Wed Feb 7 12:23:32 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.63800000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411370,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6272,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738880,"statfs":{"total":95995035648,"available":95988613120,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Feb 7 12:23:41 2024","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.38100000000000001,"15min":0.36399999999999999},"min":{"1min":0.249,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":1.397,"5min":1.397,"15min":1.397},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.40000000000000002,"15min":0.38},"min":{"1min":0.23599999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":1.1910000000000001,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.441}]},{"osd":1,"last update":"Wed Feb 7 12:23:41 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.40799999999999997,"15min":0.38600000000000001},"min":{"1min":0.315,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.4199999999999999,"5min":1.4199999999999999,"15min":1.4199999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.42199999999999999,"15min":0.40000000000000002},"min":{"1min":0.29899999999999999,"5min":0.254,"15min":0.254},"max":{"1min":1.2589999999999999,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":0.46999999999999997}]},{"osd":3,"last update":"Wed Feb 7 12:22:59 2024","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.47899999999999998,"15min":0.48099999999999998},"min":{"1min":0.34599999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.627,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.442,"5min":0.435,"15min":0.434},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.39500000000000002}]},{"osd":4,"last update":"Wed Feb 7 12:23:24 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.55700000000000005,"15min":0.54700000000000004},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.1299999999999999,"5min":1.1299999999999999,"15min":1.1299999999999999},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.58899999999999997,"15min":0.57899999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.214,"5min":1.214,"15min":1.214},"last":0.55300000000000005}]},{"osd":5,"last update":"Wed Feb 7 12:23:47 2024","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.60099999999999998,"15min":0.59299999999999997},"min":{"1min":0.41899999999999998,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.0760000000000001,"5min":1.0760000000000001,"15min":1.0760000000000001},"last":0.502},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.63600000000000001,"15min":0.628},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.3260000000000001,"5min":1.3260000000000001,"15min":1.3260000000000001},"last":0.57399999999999995}]},{"osd":6,"last update":"Wed Feb 7 12:23:10 2024","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.626},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.63400000000000001}]},{"osd":7,"last update":"Wed Feb 7 12:23:33 2024","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.3740000000000001,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.2390000000000001,"5min":1.2390000000000001,"15min":1.2390000000000001},"last":0.67600000000000005}]}]},{"osd":3,"up_from":25,"seq":107374182437,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5696,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739456,"statfs":{"total":95995035648,"available":95989202944,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Feb 7 12:23:02 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.40000000000000002,"15min":0.40300000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.56799999999999995,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.42599999999999999,"15min":0.42299999999999999},"min":{"1min":0.28799999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.46500000000000002}]},{"osd":1,"last update":"Wed Feb 7 12:23:02 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.437,"15min":0.439},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.433},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41999999999999998,"15min":0.42099999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.48999999999999999}]},{"osd":2,"last update":"Wed Feb 7 12:23:02 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.46200000000000002,"15min":0.46400000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.45700000000000002,"15min":0.45900000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.504}]},{"osd":4,"last update":"Wed Feb 7 12:23:22 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.56599999999999995,"15min":0.57099999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.73799999999999999,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.76800000000000002},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.57999999999999996,"15min":0.58399999999999996},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.77400000000000002,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.88400000000000001}]},{"osd":5,"last update":"Wed Feb 7 12:23:49 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.59999999999999998,"15min":0.59899999999999998},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.75900000000000001,"5min":0.877,"15min":0.877},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.57499999999999996,"15min":0.57199999999999995},"min":{"1min":0.42999999999999999,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.86299999999999999}]},{"osd":6,"last update":"Wed Feb 7 12:23:10 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.85599999999999998}]},{"osd":7,"last update":"Wed Feb 7 12:23:37 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.78700000000000003},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.82699999999999996}]}]},{"osd":4,"up_from":30,"seq":128849018913,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5760,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739392,"statfs":{"total":95995035648,"available":95989137408,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Feb 7 12:23:23 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.68799999999999994,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.53200000000000003,"15min":0.54000000000000004},"min":{"1min":0.40600000000000003,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.58599999999999997,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.60899999999999999}]},{"osd":1,"last update":"Wed Feb 7 12:23:23 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.53200000000000003,"15min":0.53500000000000003},"min":{"1min":0.38500000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.63100000000000001,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.69799999999999995,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.54100000000000004}]},{"osd":2,"last update":"Wed Feb 7 12:23:23 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.55300000000000005,"15min":0.55600000000000005},"min":{"1min":0.41899999999999998,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.68799999999999994,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.55200000000000005,"15min":0.55900000000000005},"min":{"1min":0.39400000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.629,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.50700000000000001}]},{"osd":3,"last update":"Wed Feb 7 12:23:23 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.53100000000000003,"15min":0.53300000000000003},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.53100000000000003,"15min":0.53300000000000003},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.66100000000000003,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.58599999999999997}]},{"osd":5,"last update":"Wed Feb 7 12:23:45 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.39400000000000002,"15min":0.39600000000000002},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.434,"15min":0.432},"min":{"1min":0.311,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.65300000000000002}]},{"osd":6,"last update":"Wed Feb 7 12:23:07 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.55000000000000004}]},{"osd":7,"last update":"Wed Feb 7 12:23:34 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.66700000000000004}]}]},{"osd":5,"up_from":38,"seq":163208757276,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5696,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739456,"statfs":{"total":95995035648,"available":95989202944,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Feb 7 12:23:47 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.71899999999999997,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.58799999999999997,"15min":0.59999999999999998},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.69199999999999995,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.52700000000000002}]},{"osd":1,"last update":"Wed Feb 7 12:23:47 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.55500000000000005,"15min":0.55900000000000005},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.66100000000000003,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.59699999999999998,"15min":0.60499999999999998},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.74299999999999999,"5min":1.224,"15min":1.224},"last":0.498}]},{"osd":2,"last update":"Wed Feb 7 12:23:47 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.57099999999999995,"15min":0.57699999999999996},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.58899999999999997,"15min":0.59499999999999997},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.71299999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.47499999999999998}]},{"osd":3,"last update":"Wed Feb 7 12:23:47 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.58299999999999996,"15min":0.58799999999999997},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.60899999999999999,"15min":0.61499999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.66100000000000003}]},{"osd":4,"last update":"Wed Feb 7 12:23:47 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.44700000000000001,"15min":0.44400000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.70699999999999996,"5min":1.1890000000000001,"15min":1.1890000000000001},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.44600000000000001,"15min":0.441},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.56699999999999995}]},{"osd":6,"last update":"Wed Feb 7 12:23:06 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.621},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.59599999999999997}]},{"osd":7,"last update":"Wed Feb 7 12:23:31 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.67500000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-02-07T12:23:55.531 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph pg dump --format=json 2024-02-07T12:23:55.751 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/mon.a/config 2024-02-07T12:23:55.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:55 smithi019 ceph-mon[29267]: from='client.14607 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-07T12:23:55.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:55 smithi019 ceph-mon[29267]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:55.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:55 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:55.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:55 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:55.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:55 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:55.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:55 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-02-07T12:23:55.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:55 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:55.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:55 smithi019 ceph-mon[24574]: from='client.14607 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-07T12:23:55.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:55 smithi019 ceph-mon[24574]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:55.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:55 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:55.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:55 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:55.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:55 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:55.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:55 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-02-07T12:23:55.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:55 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:55.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:55 smithi172 ceph-mon[25457]: from='client.14607 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-07T12:23:55.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:55 smithi172 ceph-mon[25457]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:55.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:55 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:55.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:55 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:55.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:55 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:55.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:55 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-02-07T12:23:55.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:55 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:56.817 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:56 smithi019 ceph-mon[29267]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-02-07T12:23:56.817 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:56 smithi019 ceph-mon[29267]: Deploying daemon grafana.a on smithi172 2024-02-07T12:23:56.817 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:56 smithi019 ceph-mon[24574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-02-07T12:23:56.817 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:56 smithi019 ceph-mon[24574]: Deploying daemon grafana.a on smithi172 2024-02-07T12:23:56.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:56 smithi172 ceph-mon[25457]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-02-07T12:23:56.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:56 smithi172 ceph-mon[25457]: Deploying daemon grafana.a on smithi172 2024-02-07T12:23:57.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:57 smithi019 ceph-mon[29267]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:57.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:57 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:57.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:57 smithi019 ceph-mon[24574]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:57.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:57 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:57.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:57 smithi172 ceph-mon[25457]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:57.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:57 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:23:57.964 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:23:57.966 INFO:teuthology.orchestra.run.smithi019.stderr:dumped all 2024-02-07T12:23:58.372 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:23:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:23:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:23:58.521 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:23:58 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:23:58] "GET /metrics HTTP/1.1" 200 191492 "" "Prometheus/2.33.4" 2024-02-07T12:23:58.544 INFO:teuthology.orchestra.run.smithi019.stdout:{"pg_ready":true,"pg_map":{"version":38,"stamp":"2024-02-07T12:23:56.506727+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48908,"kb_used_data":6092,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912308,"statfs":{"total":767960285184,"available":767910203392,"internally_reserved":0,"allocated":6238208,"data_stored":3488717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002047"},"pg_stats":[{"pgid":"1.0","version":"59'128","reported_seq":266,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-02-07T12:22:46.591374+0000","last_change":"2024-02-07T12:22:33.685902+0000","last_active":"2024-02-07T12:22:46.591374+0000","last_peered":"2024-02-07T12:22:46.591374+0000","last_clean":"2024-02-07T12:22:46.591374+0000","last_became_active":"2024-02-07T12:22:33.685384+0000","last_became_peered":"2024-02-07T12:22:33.685384+0000","last_unstale":"2024-02-07T12:22:46.591374+0000","last_undegraded":"2024-02-07T12:22:46.591374+0000","last_fullsized":"2024-02-07T12:22:46.591374+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-02-07T12:20:35.685309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-02-07T12:20:35.685309+0000","last_clean_scrub_stamp":"2024-02-07T12:20:35.685309+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-02-08T17:08:04.236560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234002,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6088,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739064,"statfs":{"total":95995035648,"available":95988801536,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Feb 7 12:23:34 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.46100000000000002}]},{"osd":1,"last update":"Wed Feb 7 12:23:34 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.40500000000000003}]},{"osd":2,"last update":"Wed Feb 7 12:23:34 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.54600000000000004}]},{"osd":3,"last update":"Wed Feb 7 12:23:34 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.503}]},{"osd":4,"last update":"Wed Feb 7 12:23:34 2024","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.443}]},{"osd":5,"last update":"Wed Feb 7 12:23:34 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.63900000000000001}]},{"osd":6,"last update":"Wed Feb 7 12:23:34 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.57699999999999996}]}]},{"osd":6,"up_from":46,"seq":197568495638,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6148,"kb_used_data":1084,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739004,"statfs":{"total":95995035648,"available":95988740096,"internally_reserved":0,"allocated":1110016,"data_stored":766504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Feb 7 12:23:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.52200000000000002}]},{"osd":1,"last update":"Wed Feb 7 12:23:09 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.623},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.60099999999999998}]},{"osd":2,"last update":"Wed Feb 7 12:23:09 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.80900000000000005}]},{"osd":3,"last update":"Wed Feb 7 12:23:09 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.79600000000000004},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.74099999999999999}]},{"osd":4,"last update":"Wed Feb 7 12:23:09 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.66800000000000004}]},{"osd":5,"last update":"Wed Feb 7 12:23:09 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.77000000000000002}]},{"osd":7,"last update":"Wed Feb 7 12:23:34 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.75},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.70299999999999996}]}]},{"osd":1,"up_from":13,"seq":55834574895,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6328,"kb_used_data":560,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738824,"statfs":{"total":95995035648,"available":95988555776,"internally_reserved":0,"allocated":573440,"data_stored":237311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Feb 7 12:23:20 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.35099999999999998,"15min":0.33800000000000002},"min":{"1min":0.24099999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.56799999999999995,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.379,"15min":0.35999999999999999},"min":{"1min":0.29699999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.67100000000000004}]},{"osd":2,"last update":"Wed Feb 7 12:23:39 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.39900000000000002,"15min":0.40100000000000002},"min":{"1min":0.26600000000000001,"5min":0.246,"15min":0.246},"max":{"1min":0.59199999999999997,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.41599999999999998,"15min":0.41799999999999998},"min":{"1min":0.29099999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.61799999999999999,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.53700000000000003}]},{"osd":3,"last update":"Wed Feb 7 12:23:01 2024","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.45300000000000001,"15min":0.45200000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.47999999999999998,"15min":0.48299999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.63800000000000001}]},{"osd":4,"last update":"Wed Feb 7 12:23:24 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.57199999999999995,"15min":0.57299999999999995},"min":{"1min":0.44700000000000001,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.72799999999999998,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.40100000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.80000000000000004}]},{"osd":5,"last update":"Wed Feb 7 12:23:48 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.57299999999999995,"15min":0.56699999999999995},"min":{"1min":0.47999999999999998,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.97099999999999997},{"interface":"front","average":{"1min":0.62,"5min":0.60799999999999998,"15min":0.60599999999999998},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.90400000000000003}]},{"osd":6,"last update":"Wed Feb 7 12:23:08 2024","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.91500000000000004},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.79400000000000004}]},{"osd":7,"last update":"Wed Feb 7 12:23:31 2024","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.878},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.93899999999999995}]}]},{"osd":0,"up_from":8,"seq":34359738420,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6920,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738232,"statfs":{"total":95995035648,"available":95987949568,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Feb 7 12:23:18 2024","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.36699999999999999,"15min":0.35999999999999999},"min":{"1min":0.253,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.51600000000000001,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.376,"15min":0.34799999999999998},"min":{"1min":0.32000000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.38500000000000001}]},{"osd":2,"last update":"Wed Feb 7 12:23:38 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.41799999999999998,"15min":0.42499999999999999},"min":{"1min":0.29299999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.53400000000000003,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.436,"15min":0.44},"min":{"1min":0.28100000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.63300000000000001,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.375}]},{"osd":3,"last update":"Wed Feb 7 12:23:01 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.46999999999999997,"15min":0.47199999999999998},"min":{"1min":0.31,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.505},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.45100000000000001,"15min":0.44900000000000001},"min":{"1min":0.33300000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.436}]},{"osd":4,"last update":"Wed Feb 7 12:23:22 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.56899999999999995,"15min":0.57099999999999995},"min":{"1min":0.432,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.75900000000000001,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.59099999999999997,"15min":0.59299999999999997},"min":{"1min":0.437,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.749,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.69299999999999995}]},{"osd":5,"last update":"Wed Feb 7 12:23:46 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.61299999999999999,"15min":0.62},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.79000000000000004,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.59099999999999997,"15min":0.59199999999999997},"min":{"1min":0.45300000000000001,"5min":0.434,"15min":0.434},"max":{"1min":0.751,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.61099999999999999}]},{"osd":6,"last update":"Wed Feb 7 12:23:07 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.66200000000000003}]},{"osd":7,"last update":"Wed Feb 7 12:23:32 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.63800000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411370,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6272,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738880,"statfs":{"total":95995035648,"available":95988613120,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Feb 7 12:23:41 2024","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.38100000000000001,"15min":0.36399999999999999},"min":{"1min":0.249,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":1.397,"5min":1.397,"15min":1.397},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.40000000000000002,"15min":0.38},"min":{"1min":0.23599999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":1.1910000000000001,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.441}]},{"osd":1,"last update":"Wed Feb 7 12:23:41 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.40799999999999997,"15min":0.38600000000000001},"min":{"1min":0.315,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.4199999999999999,"5min":1.4199999999999999,"15min":1.4199999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.42199999999999999,"15min":0.40000000000000002},"min":{"1min":0.29899999999999999,"5min":0.254,"15min":0.254},"max":{"1min":1.2589999999999999,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":0.46999999999999997}]},{"osd":3,"last update":"Wed Feb 7 12:22:59 2024","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.47899999999999998,"15min":0.48099999999999998},"min":{"1min":0.34599999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.627,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.442,"5min":0.435,"15min":0.434},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.39500000000000002}]},{"osd":4,"last update":"Wed Feb 7 12:23:24 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.55700000000000005,"15min":0.54700000000000004},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.1299999999999999,"5min":1.1299999999999999,"15min":1.1299999999999999},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.58899999999999997,"15min":0.57899999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.214,"5min":1.214,"15min":1.214},"last":0.55300000000000005}]},{"osd":5,"last update":"Wed Feb 7 12:23:47 2024","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.60099999999999998,"15min":0.59299999999999997},"min":{"1min":0.41899999999999998,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.0760000000000001,"5min":1.0760000000000001,"15min":1.0760000000000001},"last":0.502},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.63600000000000001,"15min":0.628},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.3260000000000001,"5min":1.3260000000000001,"15min":1.3260000000000001},"last":0.57399999999999995}]},{"osd":6,"last update":"Wed Feb 7 12:23:10 2024","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.626},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.63400000000000001}]},{"osd":7,"last update":"Wed Feb 7 12:23:33 2024","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.3740000000000001,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.2390000000000001,"5min":1.2390000000000001,"15min":1.2390000000000001},"last":0.67600000000000005}]}]},{"osd":3,"up_from":25,"seq":107374182437,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5696,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739456,"statfs":{"total":95995035648,"available":95989202944,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Feb 7 12:23:02 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.40000000000000002,"15min":0.40300000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.56799999999999995,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.42599999999999999,"15min":0.42299999999999999},"min":{"1min":0.28799999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.46500000000000002}]},{"osd":1,"last update":"Wed Feb 7 12:23:02 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.437,"15min":0.439},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.433},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41999999999999998,"15min":0.42099999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.48999999999999999}]},{"osd":2,"last update":"Wed Feb 7 12:23:02 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.46200000000000002,"15min":0.46400000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.45700000000000002,"15min":0.45900000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.504}]},{"osd":4,"last update":"Wed Feb 7 12:23:22 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.56599999999999995,"15min":0.57099999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.73799999999999999,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.76800000000000002},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.57999999999999996,"15min":0.58399999999999996},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.77400000000000002,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.88400000000000001}]},{"osd":5,"last update":"Wed Feb 7 12:23:49 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.59999999999999998,"15min":0.59899999999999998},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.75900000000000001,"5min":0.877,"15min":0.877},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.57499999999999996,"15min":0.57199999999999995},"min":{"1min":0.42999999999999999,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.86299999999999999}]},{"osd":6,"last update":"Wed Feb 7 12:23:10 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.85599999999999998}]},{"osd":7,"last update":"Wed Feb 7 12:23:37 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.78700000000000003},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.82699999999999996}]}]},{"osd":4,"up_from":30,"seq":128849018913,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5760,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739392,"statfs":{"total":95995035648,"available":95989137408,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Feb 7 12:23:23 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.68799999999999994,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.53200000000000003,"15min":0.54000000000000004},"min":{"1min":0.40600000000000003,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.58599999999999997,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.60899999999999999}]},{"osd":1,"last update":"Wed Feb 7 12:23:23 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.53200000000000003,"15min":0.53500000000000003},"min":{"1min":0.38500000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.63100000000000001,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.69799999999999995,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.54100000000000004}]},{"osd":2,"last update":"Wed Feb 7 12:23:23 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.55300000000000005,"15min":0.55600000000000005},"min":{"1min":0.41899999999999998,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.68799999999999994,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.55200000000000005,"15min":0.55900000000000005},"min":{"1min":0.39400000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.629,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.50700000000000001}]},{"osd":3,"last update":"Wed Feb 7 12:23:23 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.53100000000000003,"15min":0.53300000000000003},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.53100000000000003,"15min":0.53300000000000003},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.66100000000000003,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.58599999999999997}]},{"osd":5,"last update":"Wed Feb 7 12:23:45 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.39400000000000002,"15min":0.39600000000000002},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.434,"15min":0.432},"min":{"1min":0.311,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.65300000000000002}]},{"osd":6,"last update":"Wed Feb 7 12:23:07 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.55000000000000004}]},{"osd":7,"last update":"Wed Feb 7 12:23:34 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.66700000000000004}]}]},{"osd":5,"up_from":38,"seq":163208757276,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5696,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739456,"statfs":{"total":95995035648,"available":95989202944,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Feb 7 12:23:47 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.71899999999999997,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.58799999999999997,"15min":0.59999999999999998},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.69199999999999995,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.52700000000000002}]},{"osd":1,"last update":"Wed Feb 7 12:23:47 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.55500000000000005,"15min":0.55900000000000005},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.66100000000000003,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.59699999999999998,"15min":0.60499999999999998},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.74299999999999999,"5min":1.224,"15min":1.224},"last":0.498}]},{"osd":2,"last update":"Wed Feb 7 12:23:47 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.57099999999999995,"15min":0.57699999999999996},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.58899999999999997,"15min":0.59499999999999997},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.71299999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.47499999999999998}]},{"osd":3,"last update":"Wed Feb 7 12:23:47 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.58299999999999996,"15min":0.58799999999999997},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.60899999999999999,"15min":0.61499999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.66100000000000003}]},{"osd":4,"last update":"Wed Feb 7 12:23:47 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.44700000000000001,"15min":0.44400000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.70699999999999996,"5min":1.1890000000000001,"15min":1.1890000000000001},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.44600000000000001,"15min":0.441},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.56699999999999995}]},{"osd":6,"last update":"Wed Feb 7 12:23:06 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.621},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.59599999999999997}]},{"osd":7,"last update":"Wed Feb 7 12:23:31 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.67500000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-02-07T12:23:58.546 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-02-07T12:23:58.546 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-02-07T12:23:58.546 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-02-07T12:23:58.546 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- ceph health --format=json 2024-02-07T12:23:58.768 INFO:teuthology.orchestra.run.smithi019.stderr:Inferring config /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/mon.a/config 2024-02-07T12:23:58.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:58 smithi019 ceph-mon[24574]: from='client.24505 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-07T12:23:58.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:23:58 smithi019 ceph-mon[24574]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:58.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:58 smithi019 ceph-mon[29267]: from='client.24505 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-07T12:23:58.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:23:58 smithi019 ceph-mon[29267]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:23:59.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:58 smithi172 ceph-mon[25457]: from='client.24505 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-07T12:23:59.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:23:58 smithi172 ceph-mon[25457]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:00.530 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:24:00.530 INFO:teuthology.orchestra.run.smithi019.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-02-07T12:24:00.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:00 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2040605009' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-02-07T12:24:00.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:00 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2040605009' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-02-07T12:24:00.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:00 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2040605009' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-02-07T12:24:01.138 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-02-07T12:24:01.138 INFO:tasks.cephadm:Setup complete, yielding 2024-02-07T12:24:01.139 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-02-07T12:24:01.149 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi019.front.sepia.ceph.com 2024-02-07T12:24:01.150 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'radosgw-admin realm create --rgw-realm=r --default' 2024-02-07T12:24:01.783 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:01 smithi019 ceph-mon[24574]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:01.783 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:01 smithi019 ceph-mon[29267]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:01.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:01 smithi172 ceph-mon[25457]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:03.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:03 smithi019 ceph-mon[24574]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:03.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:03 smithi019 ceph-mon[29267]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:03.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:03 smithi172 ceph-mon[25457]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:04.636 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:24:04.636 INFO:teuthology.orchestra.run.smithi019.stdout: "id": "097f25f6-05d6-4ccc-88c8-3331f9b90893", 2024-02-07T12:24:04.636 INFO:teuthology.orchestra.run.smithi019.stdout: "name": "r", 2024-02-07T12:24:04.636 INFO:teuthology.orchestra.run.smithi019.stdout: "current_period": "9388f6a5-9de6-400f-b57e-751de1c2cf44", 2024-02-07T12:24:04.636 INFO:teuthology.orchestra.run.smithi019.stdout: "epoch": 1 2024-02-07T12:24:04.636 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:24:04.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:04 smithi172 ceph-mon[25457]: osdmap e60: 8 total, 8 up, 8 in 2024-02-07T12:24:04.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:04 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1520726135' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-02-07T12:24:04.929 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:04 smithi019 ceph-mon[29267]: osdmap e60: 8 total, 8 up, 8 in 2024-02-07T12:24:04.929 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:04 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1520726135' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-02-07T12:24:04.929 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:04 smithi019 ceph-mon[24574]: osdmap e60: 8 total, 8 up, 8 in 2024-02-07T12:24:04.929 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:04 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1520726135' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-02-07T12:24:04.930 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:04 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[47365]: level=info ts=2024-02-07T12:24:04.815Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.001257687s 2024-02-07T12:24:05.242 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'radosgw-admin zonegroup create --rgw-zonegroup=default --master --default' 2024-02-07T12:24:05.680 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:05 smithi172 ceph-mon[25457]: pgmap v43: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:05.680 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:05 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1520726135' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-02-07T12:24:05.680 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:05 smithi172 ceph-mon[25457]: osdmap e61: 8 total, 8 up, 8 in 2024-02-07T12:24:05.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:05 smithi019 ceph-mon[29267]: pgmap v43: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:05.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:05 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1520726135' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-02-07T12:24:05.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:05 smithi019 ceph-mon[29267]: osdmap e61: 8 total, 8 up, 8 in 2024-02-07T12:24:05.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:05 smithi019 ceph-mon[24574]: pgmap v43: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:05.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:05 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1520726135' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-02-07T12:24:05.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:05 smithi019 ceph-mon[24574]: osdmap e61: 8 total, 8 up, 8 in 2024-02-07T12:24:06.735 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:06 smithi019 ceph-mon[29267]: osdmap e62: 8 total, 8 up, 8 in 2024-02-07T12:24:06.736 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:06 smithi019 ceph-mon[24574]: osdmap e62: 8 total, 8 up, 8 in 2024-02-07T12:24:06.796 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:06 smithi172 ceph-mon[25457]: osdmap e62: 8 total, 8 up, 8 in 2024-02-07T12:24:06.956 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:24:06.956 INFO:teuthology.orchestra.run.smithi019.stdout: "id": "34ec7bb2-2ff4-48e3-8b8c-decca5a90fe2", 2024-02-07T12:24:06.957 INFO:teuthology.orchestra.run.smithi019.stdout: "name": "default", 2024-02-07T12:24:06.957 INFO:teuthology.orchestra.run.smithi019.stdout: "api_name": "default", 2024-02-07T12:24:06.957 INFO:teuthology.orchestra.run.smithi019.stdout: "is_master": "true", 2024-02-07T12:24:06.957 INFO:teuthology.orchestra.run.smithi019.stdout: "endpoints": [], 2024-02-07T12:24:06.957 INFO:teuthology.orchestra.run.smithi019.stdout: "hostnames": [], 2024-02-07T12:24:06.957 INFO:teuthology.orchestra.run.smithi019.stdout: "hostnames_s3website": [], 2024-02-07T12:24:06.957 INFO:teuthology.orchestra.run.smithi019.stdout: "master_zone": "", 2024-02-07T12:24:06.957 INFO:teuthology.orchestra.run.smithi019.stdout: "zones": [], 2024-02-07T12:24:06.957 INFO:teuthology.orchestra.run.smithi019.stdout: "placement_targets": [], 2024-02-07T12:24:06.957 INFO:teuthology.orchestra.run.smithi019.stdout: "default_placement": "", 2024-02-07T12:24:06.957 INFO:teuthology.orchestra.run.smithi019.stdout: "realm_id": "097f25f6-05d6-4ccc-88c8-3331f9b90893", 2024-02-07T12:24:06.957 INFO:teuthology.orchestra.run.smithi019.stdout: "sync_policy": { 2024-02-07T12:24:06.957 INFO:teuthology.orchestra.run.smithi019.stdout: "groups": [] 2024-02-07T12:24:06.957 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:24:06.957 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:24:07.119 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:06 smithi172 systemd[1]: Starting Ceph grafana.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:24:07.446 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 podman[41536]: 2024-02-07 12:24:07.089933396 +0000 UTC m=+0.018136721 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-02-07T12:24:07.446 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 podman[41536]: 2024-02-07 12:24:07.202948017 +0000 UTC m=+0.131151329 container create fcafad07f161817f09ffd319c08bcc4c232dea458f3ae21ee3c6712d789d48be (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, release=236.1648460182, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , name=ubi8, build-date=2022-03-28T10:36:18.413762, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.buildah.version=1.24.2, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., version=8.5, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.openshift.tags=base rhel8, vcs-type=git) 2024-02-07T12:24:07.447 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 podman[41536]: 2024-02-07 12:24:07.41369303 +0000 UTC m=+0.341896343 container init fcafad07f161817f09ffd319c08bcc4c232dea458f3ae21ee3c6712d789d48be (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a, description=Ceph Grafana Container, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-03-28T10:36:18.413762, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, maintainer=Paul Cuzner , name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, release=236.1648460182, io.buildah.version=1.24.2, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, architecture=x86_64) 2024-02-07T12:24:07.447 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 podman[41536]: 2024-02-07 12:24:07.416324681 +0000 UTC m=+0.344527993 container start fcafad07f161817f09ffd319c08bcc4c232dea458f3ae21ee3c6712d789d48be (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a, io.openshift.expose-services=, name=ubi8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-03-28T10:36:18.413762, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vendor=Red Hat, Inc., architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=236.1648460182, io.buildah.version=1.24.2, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, description=Ceph Grafana Container, distribution-scope=public, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner ) 2024-02-07T12:24:07.537 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default' 2024-02-07T12:24:07.696 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-mon[25457]: pgmap v46: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:07.696 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:07.696 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:24:07.697 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:07.697 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:24:07.697 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-02-07T12:24:07.697 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-02-07T12:24:07.697 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-02-07T12:24:07.698 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-02-07T12:24:07.698 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-02-07T12:24:07.698 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-02-07T12:24:07.698 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-02-07T12:24:07.698 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-02-07T12:24:07.698 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-02-07T12:24:07.698 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-02-07T12:24:07.698 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-02-07T12:24:07.698 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-02-07T12:24:07.698 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-02-07T12:24:07.698 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="App mode production" logger=settings 2024-02-07T12:24:07.698 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-02-07T12:24:07.699 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-02-07T12:24:07.699 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-02-07T12:24:07.699 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2024-02-07T12:24:07.699 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2024-02-07T12:24:07.699 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2024-02-07T12:24:07.699 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2024-02-07T12:24:07.699 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2024-02-07T12:24:07.699 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2024-02-07T12:24:07.699 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2024-02-07T12:24:07.699 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2024-02-07T12:24:07.699 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2024-02-07T12:24:07.699 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2024-02-07T12:24:07.699 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-02-07T12:24:07.699 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2024-02-07T12:24:07.700 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2024-02-07T12:24:07.700 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2024-02-07T12:24:07.700 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2024-02-07T12:24:07.700 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2024-02-07T12:24:07.700 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2024-02-07T12:24:07.700 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2024-02-07T12:24:07.700 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2024-02-07T12:24:07.701 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2024-02-07T12:24:07.701 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2024-02-07T12:24:07.701 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2024-02-07T12:24:07.701 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2024-02-07T12:24:07.701 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2024-02-07T12:24:07.701 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2024-02-07T12:24:07.701 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2024-02-07T12:24:07.701 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2024-02-07T12:24:07.701 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2024-02-07T12:24:07.701 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2024-02-07T12:24:07.702 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-02-07T12:24:07.702 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2024-02-07T12:24:07.702 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2024-02-07T12:24:07.702 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2024-02-07T12:24:07.702 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2024-02-07T12:24:07.702 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2024-02-07T12:24:07.702 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2024-02-07T12:24:07.702 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2024-02-07T12:24:07.702 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2024-02-07T12:24:07.702 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2024-02-07T12:24:07.702 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2024-02-07T12:24:07.702 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2024-02-07T12:24:07.702 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2024-02-07T12:24:07.702 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2024-02-07T12:24:07.704 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2024-02-07T12:24:07.704 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2024-02-07T12:24:07.704 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2024-02-07T12:24:07.705 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2024-02-07T12:24:07.705 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2024-02-07T12:24:07.705 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2024-02-07T12:24:07.705 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2024-02-07T12:24:07.705 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2024-02-07T12:24:07.705 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2024-02-07T12:24:07.705 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2024-02-07T12:24:07.705 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2024-02-07T12:24:07.705 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-02-07T12:24:07.705 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2024-02-07T12:24:07.705 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2024-02-07T12:24:07.705 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2024-02-07T12:24:07.705 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2024-02-07T12:24:07.705 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2024-02-07T12:24:07.706 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 bash[41536]: fcafad07f161817f09ffd319c08bcc4c232dea458f3ae21ee3c6712d789d48be 2024-02-07T12:24:07.706 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2024-02-07T12:24:07.706 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 systemd[1]: Started Ceph grafana.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:24:07.706 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2024-02-07T12:24:07.706 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2024-02-07T12:24:07.706 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2024-02-07T12:24:07.706 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2024-02-07T12:24:07.706 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2024-02-07T12:24:07.706 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2024-02-07T12:24:07.706 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2024-02-07T12:24:07.706 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2024-02-07T12:24:07.706 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2024-02-07T12:24:07.706 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2024-02-07T12:24:07.707 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2024-02-07T12:24:07.707 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2024-02-07T12:24:07.707 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2024-02-07T12:24:07.707 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2024-02-07T12:24:07.707 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2024-02-07T12:24:07.707 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2024-02-07T12:24:07.707 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2024-02-07T12:24:07.707 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2024-02-07T12:24:07.707 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2024-02-07T12:24:07.707 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2024-02-07T12:24:07.707 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-02-07T12:24:07.707 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2024-02-07T12:24:07.707 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-02-07T12:24:07.707 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-02-07T12:24:07.707 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2024-02-07T12:24:07.707 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2024-02-07T12:24:07.708 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2024-02-07T12:24:07.708 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2024-02-07T12:24:07.708 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2024-02-07T12:24:07.708 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2024-02-07T12:24:07.708 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2024-02-07T12:24:07.708 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2024-02-07T12:24:07.708 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2024-02-07T12:24:07.708 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2024-02-07T12:24:07.708 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2024-02-07T12:24:07.708 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2024-02-07T12:24:07.708 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2024-02-07T12:24:07.708 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2024-02-07T12:24:07.708 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2024-02-07T12:24:07.709 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2024-02-07T12:24:07.709 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-02-07T12:24:07.709 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2024-02-07T12:24:07.709 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2024-02-07T12:24:07.709 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2024-02-07T12:24:07.709 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2024-02-07T12:24:07.709 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2024-02-07T12:24:07.709 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2024-02-07T12:24:07.709 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2024-02-07T12:24:07.709 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2024-02-07T12:24:07.709 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2024-02-07T12:24:07.709 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2024-02-07T12:24:07.709 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2024-02-07T12:24:07.709 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2024-02-07T12:24:07.710 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2024-02-07T12:24:07.710 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2024-02-07T12:24:07.710 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2024-02-07T12:24:07.710 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2024-02-07T12:24:07.710 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2024-02-07T12:24:07.710 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2024-02-07T12:24:07.710 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2024-02-07T12:24:07.710 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2024-02-07T12:24:07.710 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2024-02-07T12:24:07.710 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2024-02-07T12:24:07.710 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2024-02-07T12:24:07.710 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2024-02-07T12:24:07.711 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2024-02-07T12:24:07.711 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2024-02-07T12:24:07.711 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2024-02-07T12:24:07.711 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2024-02-07T12:24:07.711 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2024-02-07T12:24:07.711 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2024-02-07T12:24:07.711 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2024-02-07T12:24:07.711 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2024-02-07T12:24:07.711 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2024-02-07T12:24:07.711 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-02-07T12:24:07.711 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2024-02-07T12:24:07.711 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2024-02-07T12:24:07.711 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2024-02-07T12:24:07.711 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2024-02-07T12:24:07.712 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2024-02-07T12:24:07.712 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-02-07T12:24:07.712 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2024-02-07T12:24:07.712 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2024-02-07T12:24:07.712 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2024-02-07T12:24:07.712 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2024-02-07T12:24:07.712 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-02-07T12:24:07.712 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2024-02-07T12:24:07.712 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2024-02-07T12:24:07.713 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2024-02-07T12:24:07.713 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2024-02-07T12:24:07.713 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2024-02-07T12:24:07.713 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2024-02-07T12:24:07.713 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2024-02-07T12:24:07.713 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2024-02-07T12:24:07.713 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2024-02-07T12:24:07.713 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2024-02-07T12:24:07.713 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2024-02-07T12:24:07.713 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2024-02-07T12:24:07.713 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2024-02-07T12:24:07.713 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2024-02-07T12:24:07.713 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2024-02-07T12:24:07.713 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2024-02-07T12:24:07.713 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2024-02-07T12:24:07.714 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2024-02-07T12:24:07.714 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2024-02-07T12:24:07.714 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2024-02-07T12:24:07.714 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2024-02-07T12:24:07.714 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-02-07T12:24:07.714 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-02-07T12:24:07.714 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-02-07T12:24:07.714 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2024-02-07T12:24:07.714 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-02-07T12:24:07.714 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2024-02-07T12:24:07.714 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2024-02-07T12:24:07.714 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2024-02-07T12:24:07.714 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2024-02-07T12:24:07.715 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2024-02-07T12:24:07.715 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2024-02-07T12:24:07.715 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2024-02-07T12:24:07.715 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2024-02-07T12:24:07.715 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2024-02-07T12:24:07.715 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2024-02-07T12:24:07.715 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2024-02-07T12:24:07.715 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2024-02-07T12:24:07.715 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2024-02-07T12:24:07.715 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2024-02-07T12:24:07.715 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-02-07T12:24:07.715 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2024-02-07T12:24:07.715 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2024-02-07T12:24:07.715 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2024-02-07T12:24:07.716 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2024-02-07T12:24:07.716 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2024-02-07T12:24:07.716 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2024-02-07T12:24:07.716 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2024-02-07T12:24:07.716 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2024-02-07T12:24:07.716 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2024-02-07T12:24:07.716 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2024-02-07T12:24:07.716 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2024-02-07T12:24:07.716 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2024-02-07T12:24:07.716 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2024-02-07T12:24:07.716 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2024-02-07T12:24:07.716 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2024-02-07T12:24:07.716 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2024-02-07T12:24:07.717 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2024-02-07T12:24:07.717 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2024-02-07T12:24:07.717 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2024-02-07T12:24:07.717 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2024-02-07T12:24:07.717 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2024-02-07T12:24:07.717 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2024-02-07T12:24:07.717 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-02-07T12:24:07.717 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-02-07T12:24:07.717 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2024-02-07T12:24:07.717 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-02-07T12:24:07.717 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2024-02-07T12:24:07.717 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2024-02-07T12:24:07.717 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2024-02-07T12:24:07.717 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2024-02-07T12:24:07.718 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2024-02-07T12:24:07.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:07 smithi019 ceph-mon[24574]: pgmap v46: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:07.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:07 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:07.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:07 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:24:07.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:07 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:07.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:07 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:24:07.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:07 smithi019 ceph-mon[29267]: pgmap v46: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:07.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:07 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:07.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:07 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:24:07.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:07 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:07.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:07 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:24:07.974 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2024-02-07T12:24:07.974 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2024-02-07T12:24:07.974 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2024-02-07T12:24:07.974 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2024-02-07T12:24:07.975 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2024-02-07T12:24:07.975 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2024-02-07T12:24:07.975 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2024-02-07T12:24:07.975 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2024-02-07T12:24:07.975 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-02-07T12:24:07.975 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-02-07T12:24:07.975 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2024-02-07T12:24:07.975 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2024-02-07T12:24:07.976 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2024-02-07T12:24:07.976 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2024-02-07T12:24:07.976 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2024-02-07T12:24:07.976 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2024-02-07T12:24:07.976 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-02-07T12:24:07.976 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2024-02-07T12:24:07.976 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2024-02-07T12:24:07.978 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2024-02-07T12:24:07.978 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2024-02-07T12:24:07.978 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2024-02-07T12:24:07.978 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2024-02-07T12:24:07.978 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2024-02-07T12:24:07.978 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2024-02-07T12:24:07.978 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2024-02-07T12:24:07.979 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2024-02-07T12:24:07.979 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2024-02-07T12:24:07.979 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2024-02-07T12:24:07.979 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2024-02-07T12:24:07.979 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2024-02-07T12:24:07.979 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2024-02-07T12:24:07.979 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2024-02-07T12:24:07.979 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2024-02-07T12:24:07.980 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2024-02-07T12:24:07.980 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2024-02-07T12:24:07.980 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2024-02-07T12:24:07.980 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2024-02-07T12:24:07.980 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2024-02-07T12:24:07.980 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2024-02-07T12:24:07.980 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2024-02-07T12:24:07.980 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2024-02-07T12:24:07.980 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2024-02-07T12:24:07.980 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2024-02-07T12:24:07.980 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2024-02-07T12:24:07.981 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-02-07T12:24:07.981 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2024-02-07T12:24:07.981 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2024-02-07T12:24:07.981 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2024-02-07T12:24:07.981 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2024-02-07T12:24:07.981 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2024-02-07T12:24:07.981 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-02-07T12:24:07.981 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2024-02-07T12:24:07.981 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2024-02-07T12:24:07.982 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2024-02-07T12:24:07.982 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2024-02-07T12:24:07.982 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2024-02-07T12:24:07.982 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2024-02-07T12:24:07.982 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2024-02-07T12:24:07.982 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2024-02-07T12:24:07.982 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2024-02-07T12:24:07.983 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2024-02-07T12:24:07.983 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2024-02-07T12:24:07.983 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2024-02-07T12:24:07.983 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2024-02-07T12:24:07.983 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2024-02-07T12:24:07.983 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2024-02-07T12:24:07.983 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2024-02-07T12:24:07.983 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2024-02-07T12:24:07.983 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2024-02-07T12:24:07.983 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2024-02-07T12:24:07.983 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2024-02-07T12:24:07.983 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2024-02-07T12:24:07.983 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2024-02-07T12:24:07.983 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2024-02-07T12:24:07.984 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2024-02-07T12:24:07.984 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2024-02-07T12:24:07.984 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2024-02-07T12:24:07.984 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2024-02-07T12:24:07.984 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2024-02-07T12:24:07.984 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2024-02-07T12:24:07.985 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2024-02-07T12:24:07.985 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-02-07T12:24:07.985 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-02-07T12:24:07.985 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2024-02-07T12:24:07.985 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2024-02-07T12:24:07.985 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2024-02-07T12:24:07.985 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-02-07T12:24:07.985 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2024-02-07T12:24:07.986 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2024-02-07T12:24:07.986 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-02-07T12:24:07.986 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2024-02-07T12:24:07.986 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2024-02-07T12:24:07.986 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2024-02-07T12:24:07.986 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-02-07T12:24:07.986 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2024-02-07T12:24:07.986 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2024-02-07T12:24:07.986 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2024-02-07T12:24:07.986 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2024-02-07T12:24:07.986 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-02-07T12:24:07.986 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2024-02-07T12:24:07.986 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2024-02-07T12:24:07.986 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2024-02-07T12:24:07.987 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2024-02-07T12:24:07.987 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2024-02-07T12:24:07.987 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-02-07T12:24:07.987 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2024-02-07T12:24:07.987 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2024-02-07T12:24:07.987 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-02-07T12:24:07.987 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2024-02-07T12:24:07.987 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-02-07T12:24:07.988 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-02-07T12:24:07.988 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2024-02-07T12:24:07.988 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2024-02-07T12:24:07.988 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2024-02-07T12:24:07.989 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2024-02-07T12:24:07.989 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2024-02-07T12:24:07.990 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2024-02-07T12:24:07.990 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-02-07T12:24:07.990 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2024-02-07T12:24:07.990 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2024-02-07T12:24:07.990 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2024-02-07T12:24:07.990 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2024-02-07T12:24:07.990 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2024-02-07T12:24:07.990 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2024-02-07T12:24:07.991 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2024-02-07T12:24:07.991 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2024-02-07T12:24:07.991 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2024-02-07T12:24:07.991 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2024-02-07T12:24:07.991 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2024-02-07T12:24:07.991 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2024-02-07T12:24:07.992 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2024-02-07T12:24:07.992 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2024-02-07T12:24:07.992 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2024-02-07T12:24:07.992 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2024-02-07T12:24:07.992 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2024-02-07T12:24:07.992 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2024-02-07T12:24:07.992 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2024-02-07T12:24:07.992 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2024-02-07T12:24:07.993 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2024-02-07T12:24:07.993 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2024-02-07T12:24:07.993 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2024-02-07T12:24:07.993 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2024-02-07T12:24:07.993 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2024-02-07T12:24:07.993 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2024-02-07T12:24:07.993 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2024-02-07T12:24:07.993 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2024-02-07T12:24:07.993 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2024-02-07T12:24:07.993 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2024-02-07T12:24:07.993 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2024-02-07T12:24:07.993 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2024-02-07T12:24:07.993 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2024-02-07T12:24:07.993 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2024-02-07T12:24:07.993 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2024-02-07T12:24:07.994 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2024-02-07T12:24:07.994 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2024-02-07T12:24:07.994 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2024-02-07T12:24:07.994 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2024-02-07T12:24:07.994 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2024-02-07T12:24:07.994 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2024-02-07T12:24:07.994 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2024-02-07T12:24:07.994 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2024-02-07T12:24:07.994 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=474.771564ms 2024-02-07T12:24:07.994 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-02-07T12:24:07.994 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-02-07T12:24:07.994 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-02-07T12:24:07.994 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-02-07T12:24:07.994 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-02-07T12:24:07.995 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:07+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-02-07T12:24:08.364 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:24:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:24:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:24:08.364 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:08 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:08+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-02-07T12:24:08.364 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:08 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:08+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-02-07T12:24:08.364 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:08 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:08+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-02-07T12:24:08.364 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:08 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:08+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-02-07T12:24:08.364 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:24:08 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:24:08+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-02-07T12:24:08.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:24:08 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:24:08] "GET /metrics HTTP/1.1" 200 191496 "" "Prometheus/2.33.4" 2024-02-07T12:24:08.933 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:08 smithi172 ceph-mon[25457]: pgmap v47: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 1.8 KiB/s wr, 2 op/s 2024-02-07T12:24:09.043 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:08 smithi019 ceph-mon[29267]: pgmap v47: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 1.8 KiB/s wr, 2 op/s 2024-02-07T12:24:09.043 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:08 smithi019 ceph-mon[24574]: pgmap v47: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 1.8 KiB/s wr, 2 op/s 2024-02-07T12:24:09.231 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:24:09.231 INFO:teuthology.orchestra.run.smithi019.stdout: "id": "3215871d-e33e-4ba2-aab2-4f2606ecd1bb", 2024-02-07T12:24:09.232 INFO:teuthology.orchestra.run.smithi019.stdout: "name": "z", 2024-02-07T12:24:09.232 INFO:teuthology.orchestra.run.smithi019.stdout: "domain_root": "z.rgw.meta:root", 2024-02-07T12:24:09.232 INFO:teuthology.orchestra.run.smithi019.stdout: "control_pool": "z.rgw.control", 2024-02-07T12:24:09.232 INFO:teuthology.orchestra.run.smithi019.stdout: "gc_pool": "z.rgw.log:gc", 2024-02-07T12:24:09.232 INFO:teuthology.orchestra.run.smithi019.stdout: "lc_pool": "z.rgw.log:lc", 2024-02-07T12:24:09.232 INFO:teuthology.orchestra.run.smithi019.stdout: "log_pool": "z.rgw.log", 2024-02-07T12:24:09.232 INFO:teuthology.orchestra.run.smithi019.stdout: "intent_log_pool": "z.rgw.log:intent", 2024-02-07T12:24:09.232 INFO:teuthology.orchestra.run.smithi019.stdout: "usage_log_pool": "z.rgw.log:usage", 2024-02-07T12:24:09.232 INFO:teuthology.orchestra.run.smithi019.stdout: "roles_pool": "z.rgw.meta:roles", 2024-02-07T12:24:09.233 INFO:teuthology.orchestra.run.smithi019.stdout: "reshard_pool": "z.rgw.log:reshard", 2024-02-07T12:24:09.233 INFO:teuthology.orchestra.run.smithi019.stdout: "user_keys_pool": "z.rgw.meta:users.keys", 2024-02-07T12:24:09.233 INFO:teuthology.orchestra.run.smithi019.stdout: "user_email_pool": "z.rgw.meta:users.email", 2024-02-07T12:24:09.233 INFO:teuthology.orchestra.run.smithi019.stdout: "user_swift_pool": "z.rgw.meta:users.swift", 2024-02-07T12:24:09.233 INFO:teuthology.orchestra.run.smithi019.stdout: "user_uid_pool": "z.rgw.meta:users.uid", 2024-02-07T12:24:09.233 INFO:teuthology.orchestra.run.smithi019.stdout: "otp_pool": "z.rgw.otp", 2024-02-07T12:24:09.233 INFO:teuthology.orchestra.run.smithi019.stdout: "system_key": { 2024-02-07T12:24:09.233 INFO:teuthology.orchestra.run.smithi019.stdout: "access_key": "", 2024-02-07T12:24:09.233 INFO:teuthology.orchestra.run.smithi019.stdout: "secret_key": "" 2024-02-07T12:24:09.233 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:24:09.233 INFO:teuthology.orchestra.run.smithi019.stdout: "placement_pools": [ 2024-02-07T12:24:09.233 INFO:teuthology.orchestra.run.smithi019.stdout: { 2024-02-07T12:24:09.233 INFO:teuthology.orchestra.run.smithi019.stdout: "key": "default-placement", 2024-02-07T12:24:09.233 INFO:teuthology.orchestra.run.smithi019.stdout: "val": { 2024-02-07T12:24:09.234 INFO:teuthology.orchestra.run.smithi019.stdout: "index_pool": "z.rgw.buckets.index", 2024-02-07T12:24:09.234 INFO:teuthology.orchestra.run.smithi019.stdout: "storage_classes": { 2024-02-07T12:24:09.234 INFO:teuthology.orchestra.run.smithi019.stdout: "STANDARD": { 2024-02-07T12:24:09.234 INFO:teuthology.orchestra.run.smithi019.stdout: "data_pool": "z.rgw.buckets.data" 2024-02-07T12:24:09.234 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:24:09.234 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:24:09.234 INFO:teuthology.orchestra.run.smithi019.stdout: "data_extra_pool": "z.rgw.buckets.non-ec", 2024-02-07T12:24:09.234 INFO:teuthology.orchestra.run.smithi019.stdout: "index_type": 0 2024-02-07T12:24:09.234 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:24:09.234 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:24:09.235 INFO:teuthology.orchestra.run.smithi019.stdout: ], 2024-02-07T12:24:09.235 INFO:teuthology.orchestra.run.smithi019.stdout: "realm_id": "097f25f6-05d6-4ccc-88c8-3331f9b90893", 2024-02-07T12:24:09.235 INFO:teuthology.orchestra.run.smithi019.stdout: "notif_pool": "z.rgw.log:notif" 2024-02-07T12:24:09.235 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:24:09.841 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'radosgw-admin period update --rgw-realm=r --commit' 2024-02-07T12:24:11.749 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:11 smithi019 ceph-mon[29267]: pgmap v48: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 591 B/s rd, 1.6 KiB/s wr, 2 op/s 2024-02-07T12:24:11.749 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:11 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:11.749 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:11 smithi019 ceph-mon[24574]: pgmap v48: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 591 B/s rd, 1.6 KiB/s wr, 2 op/s 2024-02-07T12:24:11.749 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:11 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:11.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:11 smithi172 ceph-mon[25457]: pgmap v48: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 591 B/s rd, 1.6 KiB/s wr, 2 op/s 2024-02-07T12:24:11.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:11 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:12.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:12 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:12.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:12 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:12.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:12 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:13.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:13 smithi172 ceph-mon[25457]: pgmap v49: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-02-07T12:24:13.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:13 smithi172 ceph-mon[25457]: osdmap e63: 8 total, 8 up, 8 in 2024-02-07T12:24:13.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:13 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1386735184' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-02-07T12:24:13.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:13 smithi172 ceph-mon[25457]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-02-07T12:24:13.911 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:13 smithi019 ceph-mon[29267]: pgmap v49: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-02-07T12:24:13.911 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:13 smithi019 ceph-mon[29267]: osdmap e63: 8 total, 8 up, 8 in 2024-02-07T12:24:13.911 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:13 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1386735184' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-02-07T12:24:13.912 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:13 smithi019 ceph-mon[29267]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-02-07T12:24:13.912 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:13 smithi019 ceph-mon[24574]: pgmap v49: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-02-07T12:24:13.912 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:13 smithi019 ceph-mon[24574]: osdmap e63: 8 total, 8 up, 8 in 2024-02-07T12:24:13.912 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:13 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1386735184' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-02-07T12:24:13.912 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:13 smithi019 ceph-mon[24574]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-02-07T12:24:14.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:14 smithi019 ceph-mon[24574]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-02-07T12:24:14.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:14 smithi019 ceph-mon[24574]: osdmap e64: 8 total, 8 up, 8 in 2024-02-07T12:24:14.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:14 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:14.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:14 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:14.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:14 smithi019 ceph-mon[29267]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-02-07T12:24:14.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:14 smithi019 ceph-mon[29267]: osdmap e64: 8 total, 8 up, 8 in 2024-02-07T12:24:14.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:14 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:14.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:14 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:14.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:14 smithi172 ceph-mon[25457]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-02-07T12:24:14.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:14 smithi172 ceph-mon[25457]: osdmap e64: 8 total, 8 up, 8 in 2024-02-07T12:24:14.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:14 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:14.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:14 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:15.950 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:15 smithi019 ceph-mon[29267]: Reconfiguring alertmanager.a (dependencies changed)... 2024-02-07T12:24:15.950 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:15 smithi019 ceph-mon[29267]: Reconfiguring daemon alertmanager.a on smithi019 2024-02-07T12:24:15.950 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:15 smithi019 ceph-mon[29267]: pgmap v52: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-02-07T12:24:15.950 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:15 smithi019 ceph-mon[29267]: osdmap e65: 8 total, 8 up, 8 in 2024-02-07T12:24:15.950 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:15 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1386735184' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-02-07T12:24:15.950 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:15 smithi019 ceph-mon[29267]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-02-07T12:24:15.950 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:15 smithi019 ceph-mon[29267]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-02-07T12:24:15.950 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:15 smithi019 ceph-mon[24574]: Reconfiguring alertmanager.a (dependencies changed)... 2024-02-07T12:24:15.951 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:15 smithi019 ceph-mon[24574]: Reconfiguring daemon alertmanager.a on smithi019 2024-02-07T12:24:15.951 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:15 smithi019 ceph-mon[24574]: pgmap v52: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-02-07T12:24:15.951 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:15 smithi019 ceph-mon[24574]: osdmap e65: 8 total, 8 up, 8 in 2024-02-07T12:24:15.951 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:15 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1386735184' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-02-07T12:24:15.951 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:15 smithi019 ceph-mon[24574]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-02-07T12:24:15.951 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:15 smithi019 ceph-mon[24574]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-02-07T12:24:16.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:15 smithi172 ceph-mon[25457]: Reconfiguring alertmanager.a (dependencies changed)... 2024-02-07T12:24:16.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:15 smithi172 ceph-mon[25457]: Reconfiguring daemon alertmanager.a on smithi019 2024-02-07T12:24:16.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:15 smithi172 ceph-mon[25457]: pgmap v52: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-02-07T12:24:16.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:15 smithi172 ceph-mon[25457]: osdmap e65: 8 total, 8 up, 8 in 2024-02-07T12:24:16.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:15 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1386735184' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-02-07T12:24:16.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:15 smithi172 ceph-mon[25457]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-02-07T12:24:16.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:15 smithi172 ceph-mon[25457]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-02-07T12:24:17.048 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:16 smithi019 ceph-mon[29267]: osdmap e66: 8 total, 8 up, 8 in 2024-02-07T12:24:17.049 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:16 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:17.049 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:16 smithi019 ceph-mon[24574]: osdmap e66: 8 total, 8 up, 8 in 2024-02-07T12:24:17.049 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:16 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:17.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:16 smithi172 ceph-mon[25457]: osdmap e66: 8 total, 8 up, 8 in 2024-02-07T12:24:17.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:16 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:17.311 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:17 smithi019 systemd[1]: Stopping Ceph alertmanager.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:24:17.311 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:17 smithi019 bash[48940]: Error: no container with name or ID "ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager.a" found: no such container 2024-02-07T12:24:17.594 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[47365]: level=info ts=2024-02-07T12:24:17.308Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-02-07T12:24:17.594 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:17 smithi019 podman[48953]: 2024-02-07 12:24:17.329327995 +0000 UTC m=+0.133031926 container died cbb2f364c91275b458aa1bcf313840277763302e26432fd19da72c88dc670115 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-02-07T12:24:17.994 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:17 smithi172 ceph-mon[25457]: pgmap v55: 97 pgs: 64 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:17.994 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:17 smithi172 ceph-mon[25457]: osdmap e67: 8 total, 8 up, 8 in 2024-02-07T12:24:17.994 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:17 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1021656865' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-02-07T12:24:18.011 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:17 smithi019 podman[48953]: 2024-02-07 12:24:17.767208497 +0000 UTC m=+0.570912427 container cleanup cbb2f364c91275b458aa1bcf313840277763302e26432fd19da72c88dc670115 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-02-07T12:24:18.012 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:17 smithi019 bash[48953]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a 2024-02-07T12:24:18.012 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:17 smithi019 ceph-mon[29267]: pgmap v55: 97 pgs: 64 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:18.012 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:17 smithi019 ceph-mon[29267]: osdmap e67: 8 total, 8 up, 8 in 2024-02-07T12:24:18.012 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:17 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1021656865' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-02-07T12:24:18.012 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:17 smithi019 ceph-mon[24574]: pgmap v55: 97 pgs: 64 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:18.012 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:17 smithi019 ceph-mon[24574]: osdmap e67: 8 total, 8 up, 8 in 2024-02-07T12:24:18.012 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:17 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1021656865' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-02-07T12:24:18.325 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:18 smithi019 podman[48967]: 2024-02-07 12:24:18.010239234 +0000 UTC m=+0.676992249 container remove cbb2f364c91275b458aa1bcf313840277763302e26432fd19da72c88dc670115 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-02-07T12:24:18.326 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:18 smithi019 podman[48967]: 2024-02-07 12:24:18.120474387 +0000 UTC m=+0.787227401 volume remove 2167d333b0e6979e2eac0fd1b30bf645ecb4b2ce7e39a83ac8cb6b0ca1f5b7b1 2024-02-07T12:24:18.326 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:18 smithi019 bash[48981]: Error: no container with name or ID "ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager.a" found: no such container 2024-02-07T12:24:18.326 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:18 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@alertmanager.a.service: Deactivated successfully. 2024-02-07T12:24:18.326 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:18 smithi019 systemd[1]: Stopped Ceph alertmanager.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:24:18.326 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:18 smithi019 systemd[1]: Starting Ceph alertmanager.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:24:18.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:24:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:24:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:24:18.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:24:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:24:18] "GET /metrics HTTP/1.1" 200 195394 "" "Prometheus/2.33.4" 2024-02-07T12:24:18.595 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:18 smithi019 podman[49049]: 2024-02-07 12:24:18.267974552 +0000 UTC m=+0.018372947 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-02-07T12:24:18.595 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:18 smithi019 podman[49049]: 2024-02-07 12:24:18.387118912 +0000 UTC m=+0.137517348 volume create af4495cbb1d9dd67fbd15f8b3aeabc6305232a0aa10c73b0190d9ead47994e46 2024-02-07T12:24:18.595 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:18 smithi019 podman[49049]: 2024-02-07 12:24:18.462497247 +0000 UTC m=+0.212895704 container create 4eb1830b3c2021fa79a56d0078a022cb215d66579401a8c62c4771428bb779cf (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-02-07T12:24:18.889 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:18 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1021656865' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-02-07T12:24:18.889 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:18 smithi019 ceph-mon[29267]: osdmap e68: 8 total, 8 up, 8 in 2024-02-07T12:24:18.889 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:18 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1021656865' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-02-07T12:24:18.889 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:18 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1021656865' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-02-07T12:24:18.889 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:18 smithi019 ceph-mon[29267]: osdmap e69: 8 total, 8 up, 8 in 2024-02-07T12:24:18.889 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:18 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1021656865' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-02-07T12:24:18.890 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:18 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1021656865' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-02-07T12:24:18.890 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:18 smithi019 ceph-mon[24574]: osdmap e68: 8 total, 8 up, 8 in 2024-02-07T12:24:18.890 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:18 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1021656865' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-02-07T12:24:18.890 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:18 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1021656865' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-02-07T12:24:18.890 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:18 smithi019 ceph-mon[24574]: osdmap e69: 8 total, 8 up, 8 in 2024-02-07T12:24:18.890 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:18 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1021656865' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-02-07T12:24:18.890 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:18 smithi019 podman[49049]: 2024-02-07 12:24:18.765173059 +0000 UTC m=+0.515571454 container init 4eb1830b3c2021fa79a56d0078a022cb215d66579401a8c62c4771428bb779cf (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-02-07T12:24:18.890 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:18 smithi019 podman[49049]: 2024-02-07 12:24:18.768082761 +0000 UTC m=+0.518481159 container start 4eb1830b3c2021fa79a56d0078a022cb215d66579401a8c62c4771428bb779cf (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-02-07T12:24:18.890 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=info ts=2024-02-07T12:24:18.778Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-02-07T12:24:18.890 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=info ts=2024-02-07T12:24:18.778Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-02-07T12:24:18.890 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=info ts=2024-02-07T12:24:18.779Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.19 port=9094 2024-02-07T12:24:18.890 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=info ts=2024-02-07T12:24:18.781Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-02-07T12:24:18.891 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=info ts=2024-02-07T12:24:18.806Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-02-07T12:24:18.891 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=info ts=2024-02-07T12:24:18.807Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-02-07T12:24:18.891 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=info ts=2024-02-07T12:24:18.809Z caller=main.go:518 msg=Listening address=:9093 2024-02-07T12:24:18.891 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=info ts=2024-02-07T12:24:18.809Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-02-07T12:24:18.891 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:18 smithi019 bash[49049]: 4eb1830b3c2021fa79a56d0078a022cb215d66579401a8c62c4771428bb779cf 2024-02-07T12:24:18.891 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:18 smithi019 systemd[1]: Started Ceph alertmanager.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:24:18.941 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:18 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1021656865' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-02-07T12:24:18.941 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:18 smithi172 ceph-mon[25457]: osdmap e68: 8 total, 8 up, 8 in 2024-02-07T12:24:18.941 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:18 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1021656865' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-02-07T12:24:18.942 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:18 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1021656865' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-02-07T12:24:18.942 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:18 smithi172 ceph-mon[25457]: osdmap e69: 8 total, 8 up, 8 in 2024-02-07T12:24:18.942 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:18 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1021656865' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-02-07T12:24:19.694 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:24:19.694 INFO:teuthology.orchestra.run.smithi019.stdout: "id": "86d9b81c-56f2-404f-81f6-743fff99b6f6", 2024-02-07T12:24:19.694 INFO:teuthology.orchestra.run.smithi019.stdout: "epoch": 1, 2024-02-07T12:24:19.694 INFO:teuthology.orchestra.run.smithi019.stdout: "predecessor_uuid": "9388f6a5-9de6-400f-b57e-751de1c2cf44", 2024-02-07T12:24:19.694 INFO:teuthology.orchestra.run.smithi019.stdout: "sync_status": [], 2024-02-07T12:24:19.694 INFO:teuthology.orchestra.run.smithi019.stdout: "period_map": { 2024-02-07T12:24:19.695 INFO:teuthology.orchestra.run.smithi019.stdout: "id": "86d9b81c-56f2-404f-81f6-743fff99b6f6", 2024-02-07T12:24:19.695 INFO:teuthology.orchestra.run.smithi019.stdout: "zonegroups": [ 2024-02-07T12:24:19.695 INFO:teuthology.orchestra.run.smithi019.stdout: { 2024-02-07T12:24:19.695 INFO:teuthology.orchestra.run.smithi019.stdout: "id": "34ec7bb2-2ff4-48e3-8b8c-decca5a90fe2", 2024-02-07T12:24:19.695 INFO:teuthology.orchestra.run.smithi019.stdout: "name": "default", 2024-02-07T12:24:19.695 INFO:teuthology.orchestra.run.smithi019.stdout: "api_name": "default", 2024-02-07T12:24:19.695 INFO:teuthology.orchestra.run.smithi019.stdout: "is_master": "true", 2024-02-07T12:24:19.695 INFO:teuthology.orchestra.run.smithi019.stdout: "endpoints": [], 2024-02-07T12:24:19.695 INFO:teuthology.orchestra.run.smithi019.stdout: "hostnames": [], 2024-02-07T12:24:19.695 INFO:teuthology.orchestra.run.smithi019.stdout: "hostnames_s3website": [], 2024-02-07T12:24:19.695 INFO:teuthology.orchestra.run.smithi019.stdout: "master_zone": "3215871d-e33e-4ba2-aab2-4f2606ecd1bb", 2024-02-07T12:24:19.695 INFO:teuthology.orchestra.run.smithi019.stdout: "zones": [ 2024-02-07T12:24:19.695 INFO:teuthology.orchestra.run.smithi019.stdout: { 2024-02-07T12:24:19.695 INFO:teuthology.orchestra.run.smithi019.stdout: "id": "3215871d-e33e-4ba2-aab2-4f2606ecd1bb", 2024-02-07T12:24:19.696 INFO:teuthology.orchestra.run.smithi019.stdout: "name": "z", 2024-02-07T12:24:19.696 INFO:teuthology.orchestra.run.smithi019.stdout: "endpoints": [], 2024-02-07T12:24:19.696 INFO:teuthology.orchestra.run.smithi019.stdout: "log_meta": "false", 2024-02-07T12:24:19.696 INFO:teuthology.orchestra.run.smithi019.stdout: "log_data": "false", 2024-02-07T12:24:19.696 INFO:teuthology.orchestra.run.smithi019.stdout: "bucket_index_max_shards": 11, 2024-02-07T12:24:19.696 INFO:teuthology.orchestra.run.smithi019.stdout: "read_only": "false", 2024-02-07T12:24:19.696 INFO:teuthology.orchestra.run.smithi019.stdout: "tier_type": "", 2024-02-07T12:24:19.696 INFO:teuthology.orchestra.run.smithi019.stdout: "sync_from_all": "true", 2024-02-07T12:24:19.696 INFO:teuthology.orchestra.run.smithi019.stdout: "sync_from": [], 2024-02-07T12:24:19.696 INFO:teuthology.orchestra.run.smithi019.stdout: "redirect_zone": "" 2024-02-07T12:24:19.696 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:24:19.696 INFO:teuthology.orchestra.run.smithi019.stdout: ], 2024-02-07T12:24:19.696 INFO:teuthology.orchestra.run.smithi019.stdout: "placement_targets": [ 2024-02-07T12:24:19.696 INFO:teuthology.orchestra.run.smithi019.stdout: { 2024-02-07T12:24:19.697 INFO:teuthology.orchestra.run.smithi019.stdout: "name": "default-placement", 2024-02-07T12:24:19.697 INFO:teuthology.orchestra.run.smithi019.stdout: "tags": [], 2024-02-07T12:24:19.697 INFO:teuthology.orchestra.run.smithi019.stdout: "storage_classes": [ 2024-02-07T12:24:19.697 INFO:teuthology.orchestra.run.smithi019.stdout: "STANDARD" 2024-02-07T12:24:19.697 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2024-02-07T12:24:19.697 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:24:19.697 INFO:teuthology.orchestra.run.smithi019.stdout: ], 2024-02-07T12:24:19.697 INFO:teuthology.orchestra.run.smithi019.stdout: "default_placement": "default-placement", 2024-02-07T12:24:19.697 INFO:teuthology.orchestra.run.smithi019.stdout: "realm_id": "097f25f6-05d6-4ccc-88c8-3331f9b90893", 2024-02-07T12:24:19.697 INFO:teuthology.orchestra.run.smithi019.stdout: "sync_policy": { 2024-02-07T12:24:19.697 INFO:teuthology.orchestra.run.smithi019.stdout: "groups": [] 2024-02-07T12:24:19.697 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:24:19.697 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:24:19.697 INFO:teuthology.orchestra.run.smithi019.stdout: ], 2024-02-07T12:24:19.697 INFO:teuthology.orchestra.run.smithi019.stdout: "short_zone_ids": [ 2024-02-07T12:24:19.698 INFO:teuthology.orchestra.run.smithi019.stdout: { 2024-02-07T12:24:19.698 INFO:teuthology.orchestra.run.smithi019.stdout: "key": "3215871d-e33e-4ba2-aab2-4f2606ecd1bb", 2024-02-07T12:24:19.698 INFO:teuthology.orchestra.run.smithi019.stdout: "val": 2516250083 2024-02-07T12:24:19.698 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:24:19.698 INFO:teuthology.orchestra.run.smithi019.stdout: ] 2024-02-07T12:24:19.698 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:24:19.698 INFO:teuthology.orchestra.run.smithi019.stdout: "master_zonegroup": "34ec7bb2-2ff4-48e3-8b8c-decca5a90fe2", 2024-02-07T12:24:19.698 INFO:teuthology.orchestra.run.smithi019.stdout: "master_zone": "3215871d-e33e-4ba2-aab2-4f2606ecd1bb", 2024-02-07T12:24:19.698 INFO:teuthology.orchestra.run.smithi019.stdout: "period_config": { 2024-02-07T12:24:19.698 INFO:teuthology.orchestra.run.smithi019.stdout: "bucket_quota": { 2024-02-07T12:24:19.698 INFO:teuthology.orchestra.run.smithi019.stdout: "enabled": false, 2024-02-07T12:24:19.698 INFO:teuthology.orchestra.run.smithi019.stdout: "check_on_raw": false, 2024-02-07T12:24:19.698 INFO:teuthology.orchestra.run.smithi019.stdout: "max_size": -1, 2024-02-07T12:24:19.698 INFO:teuthology.orchestra.run.smithi019.stdout: "max_size_kb": 0, 2024-02-07T12:24:19.698 INFO:teuthology.orchestra.run.smithi019.stdout: "max_objects": -1 2024-02-07T12:24:19.699 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:24:19.699 INFO:teuthology.orchestra.run.smithi019.stdout: "user_quota": { 2024-02-07T12:24:19.699 INFO:teuthology.orchestra.run.smithi019.stdout: "enabled": false, 2024-02-07T12:24:19.699 INFO:teuthology.orchestra.run.smithi019.stdout: "check_on_raw": false, 2024-02-07T12:24:19.699 INFO:teuthology.orchestra.run.smithi019.stdout: "max_size": -1, 2024-02-07T12:24:19.699 INFO:teuthology.orchestra.run.smithi019.stdout: "max_size_kb": 0, 2024-02-07T12:24:19.699 INFO:teuthology.orchestra.run.smithi019.stdout: "max_objects": -1 2024-02-07T12:24:19.699 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:24:19.699 INFO:teuthology.orchestra.run.smithi019.stdout: "user_ratelimit": { 2024-02-07T12:24:19.699 INFO:teuthology.orchestra.run.smithi019.stdout: "max_read_ops": 0, 2024-02-07T12:24:19.699 INFO:teuthology.orchestra.run.smithi019.stdout: "max_write_ops": 0, 2024-02-07T12:24:19.699 INFO:teuthology.orchestra.run.smithi019.stdout: "max_read_bytes": 0, 2024-02-07T12:24:19.699 INFO:teuthology.orchestra.run.smithi019.stdout: "max_write_bytes": 0, 2024-02-07T12:24:19.699 INFO:teuthology.orchestra.run.smithi019.stdout: "enabled": false 2024-02-07T12:24:19.699 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:24:19.700 INFO:teuthology.orchestra.run.smithi019.stdout: "bucket_ratelimit": { 2024-02-07T12:24:19.700 INFO:teuthology.orchestra.run.smithi019.stdout: "max_read_ops": 0, 2024-02-07T12:24:19.700 INFO:teuthology.orchestra.run.smithi019.stdout: "max_write_ops": 0, 2024-02-07T12:24:19.700 INFO:teuthology.orchestra.run.smithi019.stdout: "max_read_bytes": 0, 2024-02-07T12:24:19.700 INFO:teuthology.orchestra.run.smithi019.stdout: "max_write_bytes": 0, 2024-02-07T12:24:19.700 INFO:teuthology.orchestra.run.smithi019.stdout: "enabled": false 2024-02-07T12:24:19.700 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:24:19.700 INFO:teuthology.orchestra.run.smithi019.stdout: "anonymous_ratelimit": { 2024-02-07T12:24:19.700 INFO:teuthology.orchestra.run.smithi019.stdout: "max_read_ops": 0, 2024-02-07T12:24:19.700 INFO:teuthology.orchestra.run.smithi019.stdout: "max_write_ops": 0, 2024-02-07T12:24:19.700 INFO:teuthology.orchestra.run.smithi019.stdout: "max_read_bytes": 0, 2024-02-07T12:24:19.700 INFO:teuthology.orchestra.run.smithi019.stdout: "max_write_bytes": 0, 2024-02-07T12:24:19.700 INFO:teuthology.orchestra.run.smithi019.stdout: "enabled": false 2024-02-07T12:24:19.700 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:24:19.701 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:24:19.701 INFO:teuthology.orchestra.run.smithi019.stdout: "realm_id": "097f25f6-05d6-4ccc-88c8-3331f9b90893", 2024-02-07T12:24:19.701 INFO:teuthology.orchestra.run.smithi019.stdout: "realm_name": "r", 2024-02-07T12:24:19.701 INFO:teuthology.orchestra.run.smithi019.stdout: "realm_epoch": 2 2024-02-07T12:24:19.701 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:24:19.987 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:19 smithi019 ceph-mon[24574]: pgmap v58: 129 pgs: 32 creating+peering, 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-02-07T12:24:19.987 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:19 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:19.987 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:19 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1021656865' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-02-07T12:24:19.987 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:19 smithi019 ceph-mon[24574]: osdmap e70: 8 total, 8 up, 8 in 2024-02-07T12:24:19.987 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:19 smithi019 ceph-mon[29267]: pgmap v58: 129 pgs: 32 creating+peering, 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-02-07T12:24:19.987 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:19 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:19.987 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:19 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1021656865' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-02-07T12:24:19.987 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:19 smithi019 ceph-mon[29267]: osdmap e70: 8 total, 8 up, 8 in 2024-02-07T12:24:20.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:19 smithi172 ceph-mon[25457]: pgmap v58: 129 pgs: 32 creating+peering, 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-02-07T12:24:20.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:19 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:20.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:19 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1021656865' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-02-07T12:24:20.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:19 smithi172 ceph-mon[25457]: osdmap e70: 8 total, 8 up, 8 in 2024-02-07T12:24:20.450 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000' 2024-02-07T12:24:20.772 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:20 smithi019 ceph-mon[29267]: Reconfiguring prometheus.a (dependencies changed)... 2024-02-07T12:24:20.772 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:20 smithi019 ceph-mon[29267]: Reconfiguring daemon prometheus.a on smithi172 2024-02-07T12:24:20.773 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:20 smithi019 ceph-mon[24574]: Reconfiguring prometheus.a (dependencies changed)... 2024-02-07T12:24:20.773 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:20 smithi019 ceph-mon[24574]: Reconfiguring daemon prometheus.a on smithi172 2024-02-07T12:24:21.022 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:20 smithi172 ceph-mon[25457]: Reconfiguring prometheus.a (dependencies changed)... 2024-02-07T12:24:21.022 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:20 smithi172 ceph-mon[25457]: Reconfiguring daemon prometheus.a on smithi172 2024-02-07T12:24:21.066 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:20 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=info ts=2024-02-07T12:24:20.781Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000498499s 2024-02-07T12:24:21.806 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:21 smithi019 ceph-mon[29267]: pgmap v61: 129 pgs: 32 creating+peering, 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-02-07T12:24:21.806 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:21 smithi019 ceph-mon[24574]: pgmap v61: 129 pgs: 32 creating+peering, 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-02-07T12:24:21.830 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:21 smithi172 ceph-mon[25457]: pgmap v61: 129 pgs: 32 creating+peering, 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-02-07T12:24:22.384 INFO:teuthology.orchestra.run.smithi019.stdout:Scheduled rgw.foo update... 2024-02-07T12:24:23.052 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph osd pool create foo' 2024-02-07T12:24:23.572 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:23 smithi172 ceph-mon[25457]: from='client.14685 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:24:23.573 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:23 smithi172 ceph-mon[25457]: Saving service rgw.foo spec with placement count:2 2024-02-07T12:24:23.573 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:23 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:23.573 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:23 smithi172 ceph-mon[25457]: pgmap v62: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.5 KiB/s wr, 4 op/s 2024-02-07T12:24:23.688 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:23 smithi019 ceph-mon[29267]: from='client.14685 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:24:23.688 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:23 smithi019 ceph-mon[29267]: Saving service rgw.foo spec with placement count:2 2024-02-07T12:24:23.688 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:23 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:23.689 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:23 smithi019 ceph-mon[29267]: pgmap v62: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.5 KiB/s wr, 4 op/s 2024-02-07T12:24:23.689 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:23 smithi019 ceph-mon[24574]: from='client.14685 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:24:23.689 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:23 smithi019 ceph-mon[24574]: Saving service rgw.foo spec with placement count:2 2024-02-07T12:24:23.689 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:23 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:23.689 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:23 smithi019 ceph-mon[24574]: pgmap v62: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.5 KiB/s wr, 4 op/s 2024-02-07T12:24:24.370 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:23 smithi172 systemd[1]: Stopping Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:24:24.370 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:23 smithi172 bash[42227]: Error: no container with name or ID "ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus.a" found: no such container 2024-02-07T12:24:24.370 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:24 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:24:24.102Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-02-07T12:24:24.370 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:24 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:24:24.102Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-02-07T12:24:24.370 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:24 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:24:24.102Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-02-07T12:24:24.370 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:24 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:24:24.102Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-02-07T12:24:24.370 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:24 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:24:24.102Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-02-07T12:24:24.370 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:24 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:24:24.102Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-02-07T12:24:24.371 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:24 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:24:24.102Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-02-07T12:24:24.371 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:24 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:24:24.102Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-02-07T12:24:24.371 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:24 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:24:24.102Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-02-07T12:24:24.371 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:24 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:24:24.102Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-02-07T12:24:24.371 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:24 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:24:24.102Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-02-07T12:24:24.371 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:24 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[40580]: ts=2024-02-07T12:24:24.102Z caller=main.go:1066 level=info msg="See you next time!" 2024-02-07T12:24:24.371 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:24 smithi172 podman[42238]: 2024-02-07 12:24:24.11251886 +0000 UTC m=+0.116452020 container died 244f779cb86f73c1299edc3e526cd98a829eb895cd0022d1104459f3b6139037 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:24:24.780 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:24 smithi172 podman[42238]: 2024-02-07 12:24:24.551925125 +0000 UTC m=+0.555858285 container cleanup 244f779cb86f73c1299edc3e526cd98a829eb895cd0022d1104459f3b6139037 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:24:24.780 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:24 smithi172 bash[42238]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a 2024-02-07T12:24:25.048 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:24 smithi172 podman[42252]: 2024-02-07 12:24:24.778270351 +0000 UTC m=+0.667960364 container remove 244f779cb86f73c1299edc3e526cd98a829eb895cd0022d1104459f3b6139037 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:24:25.049 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:24 smithi172 bash[42264]: Error: no container with name or ID "ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus.a" found: no such container 2024-02-07T12:24:25.049 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:24 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@prometheus.a.service: Deactivated successfully. 2024-02-07T12:24:25.049 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:24 smithi172 systemd[1]: Stopped Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:24:25.049 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:24 smithi172 systemd[1]: Starting Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:24:25.350 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:25 smithi172 podman[42333]: 2024-02-07 12:24:24.950594518 +0000 UTC m=+0.018240670 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-02-07T12:24:25.350 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:25 smithi172 podman[42333]: 2024-02-07 12:24:25.054014403 +0000 UTC m=+0.121660553 container create d482181f6c267d79bfe49078ab0879db2c42fd66c34f44b695cfaf633dfa89a8 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:24:25.350 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:25 smithi172 podman[42333]: 2024-02-07 12:24:25.264886841 +0000 UTC m=+0.332532992 container init d482181f6c267d79bfe49078ab0879db2c42fd66c34f44b695cfaf633dfa89a8 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:24:25.350 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:25 smithi172 podman[42333]: 2024-02-07 12:24:25.267456938 +0000 UTC m=+0.335103090 container start d482181f6c267d79bfe49078ab0879db2c42fd66c34f44b695cfaf633dfa89a8 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:24:25.350 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:24:25.290Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-02-07T12:24:25.350 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:24:25.290Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-02-07T12:24:25.350 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:24:25.290Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-02-07T12:24:25.350 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:24:25.290Z caller=main.go:518 level=info host_details="(Linux 5.14.0-418.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Mon Feb 5 23:17:48 UTC 2024 x86_64 smithi172 (none))" 2024-02-07T12:24:25.351 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:24:25.290Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-02-07T12:24:25.351 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:24:25.290Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-02-07T12:24:25.351 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:24:25.291Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-02-07T12:24:25.351 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:24:25.292Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-02-07T12:24:25.351 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:24:25.293Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-02-07T12:24:25.351 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:24:25.295Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-02-07T12:24:25.351 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:24:25.295Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.161µs 2024-02-07T12:24:25.351 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:24:25.295Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-02-07T12:24:25.351 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:25 smithi172 bash[42333]: d482181f6c267d79bfe49078ab0879db2c42fd66c34f44b695cfaf633dfa89a8 2024-02-07T12:24:25.588 INFO:teuthology.orchestra.run.smithi019.stderr:pool 'foo' created 2024-02-07T12:24:25.619 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:25 smithi172 systemd[1]: Started Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:24:25.619 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:24:25.497Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-02-07T12:24:25.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:24:25.497Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-02-07T12:24:25.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:24:25.497Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=25.543µs wal_replay_duration=202.161879ms total_replay_duration=202.201942ms 2024-02-07T12:24:25.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:24:25.500Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-02-07T12:24:25.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:24:25.500Z caller=main.go:947 level=info msg="TSDB started" 2024-02-07T12:24:25.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:24:25.500Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-02-07T12:24:25.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:24:25.518Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=18.43662ms db_storage=647ns remote_storage=1.803µs web_handler=372ns query_engine=778ns scrape=200.007µs scrape_sd=28.945µs notify=44.574µs notify_sd=9.677µs rules=17.848469ms 2024-02-07T12:24:25.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:24:25.518Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-02-07T12:24:25.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-mon[25457]: pgmap v63: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.0 KiB/s wr, 2 op/s 2024-02-07T12:24:25.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2853891699' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-02-07T12:24:25.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-mon[25457]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-02-07T12:24:25.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:25.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:24:25.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.19:9093"}]: dispatch 2024-02-07T12:24:25.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:25.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:24:25.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.172:9095"}]: dispatch 2024-02-07T12:24:25.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:25.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:24:25.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.172:3000"}]: dispatch 2024-02-07T12:24:25.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:25.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:24:25.622 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:25.622 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:25 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:24:25.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[29267]: pgmap v63: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.0 KiB/s wr, 2 op/s 2024-02-07T12:24:25.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2853891699' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-02-07T12:24:25.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[29267]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-02-07T12:24:25.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:25.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:24:25.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.19:9093"}]: dispatch 2024-02-07T12:24:25.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:25.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:24:25.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.172:9095"}]: dispatch 2024-02-07T12:24:25.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:25.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:24:25.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.172:3000"}]: dispatch 2024-02-07T12:24:25.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:25.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:24:25.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:25.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:24:25.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[24574]: pgmap v63: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.0 KiB/s wr, 2 op/s 2024-02-07T12:24:25.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2853891699' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-02-07T12:24:25.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[24574]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-02-07T12:24:25.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:25.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:24:25.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.19:9093"}]: dispatch 2024-02-07T12:24:25.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:25.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:24:25.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.172:9095"}]: dispatch 2024-02-07T12:24:25.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:25.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:24:25.847 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.172:3000"}]: dispatch 2024-02-07T12:24:25.847 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:25.847 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:24:25.847 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:25.847 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:25 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:24:26.209 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'rbd pool init foo' 2024-02-07T12:24:26.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:26 smithi019 ceph-mon[24574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:24:26.848 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:26 smithi019 ceph-mon[24574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.19:9093"}]: dispatch 2024-02-07T12:24:26.848 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:26 smithi019 ceph-mon[24574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:24:26.848 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:26 smithi019 ceph-mon[24574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.172:9095"}]: dispatch 2024-02-07T12:24:26.848 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:26 smithi019 ceph-mon[24574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:24:26.848 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:26 smithi019 ceph-mon[24574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.172:3000"}]: dispatch 2024-02-07T12:24:26.848 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:26 smithi019 ceph-mon[24574]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-02-07T12:24:26.848 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:26 smithi019 ceph-mon[24574]: osdmap e71: 8 total, 8 up, 8 in 2024-02-07T12:24:26.848 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:26 smithi019 ceph-mon[29267]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:24:26.848 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:26 smithi019 ceph-mon[29267]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.19:9093"}]: dispatch 2024-02-07T12:24:26.848 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:26 smithi019 ceph-mon[29267]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:24:26.848 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:26 smithi019 ceph-mon[29267]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.172:9095"}]: dispatch 2024-02-07T12:24:26.849 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:26 smithi019 ceph-mon[29267]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:24:26.849 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:26 smithi019 ceph-mon[29267]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.172:3000"}]: dispatch 2024-02-07T12:24:26.849 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:26 smithi019 ceph-mon[29267]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-02-07T12:24:26.849 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:26 smithi019 ceph-mon[29267]: osdmap e71: 8 total, 8 up, 8 in 2024-02-07T12:24:26.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:26 smithi172 ceph-mon[25457]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:24:26.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:26 smithi172 ceph-mon[25457]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.19:9093"}]: dispatch 2024-02-07T12:24:26.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:26 smithi172 ceph-mon[25457]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:24:26.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:26 smithi172 ceph-mon[25457]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.172:9095"}]: dispatch 2024-02-07T12:24:26.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:26 smithi172 ceph-mon[25457]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:24:26.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:26 smithi172 ceph-mon[25457]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.172:3000"}]: dispatch 2024-02-07T12:24:26.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:26 smithi172 ceph-mon[25457]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-02-07T12:24:26.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:26 smithi172 ceph-mon[25457]: osdmap e71: 8 total, 8 up, 8 in 2024-02-07T12:24:27.843 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:27 smithi172 ceph-mon[25457]: pgmap v65: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 908 B/s rd, 908 B/s wr, 2 op/s 2024-02-07T12:24:27.843 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:27 smithi172 ceph-mon[25457]: osdmap e72: 8 total, 8 up, 8 in 2024-02-07T12:24:27.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:27 smithi019 ceph-mon[24574]: pgmap v65: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 908 B/s rd, 908 B/s wr, 2 op/s 2024-02-07T12:24:27.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:27 smithi019 ceph-mon[24574]: osdmap e72: 8 total, 8 up, 8 in 2024-02-07T12:24:27.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:27 smithi019 ceph-mon[29267]: pgmap v65: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 908 B/s rd, 908 B/s wr, 2 op/s 2024-02-07T12:24:27.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:27 smithi019 ceph-mon[29267]: osdmap e72: 8 total, 8 up, 8 in 2024-02-07T12:24:28.786 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:28 smithi019 ceph-mon[29267]: osdmap e73: 8 total, 8 up, 8 in 2024-02-07T12:24:28.786 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:28 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1119255444' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-02-07T12:24:28.786 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:28 smithi019 ceph-mon[29267]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-02-07T12:24:28.786 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:28 smithi019 ceph-mon[24574]: osdmap e73: 8 total, 8 up, 8 in 2024-02-07T12:24:28.786 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:28 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1119255444' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-02-07T12:24:28.786 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:28 smithi019 ceph-mon[24574]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-02-07T12:24:28.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:28 smithi172 ceph-mon[25457]: osdmap e73: 8 total, 8 up, 8 in 2024-02-07T12:24:28.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:28 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1119255444' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-02-07T12:24:28.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:28 smithi172 ceph-mon[25457]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-02-07T12:24:29.072 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:24:28 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=info ts=2024-02-07T12:24:28.783Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002302583s 2024-02-07T12:24:29.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:29 smithi019 ceph-mon[24574]: pgmap v68: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:29.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:29 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:29.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:29 smithi019 ceph-mon[24574]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-02-07T12:24:29.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:29 smithi019 ceph-mon[24574]: osdmap e74: 8 total, 8 up, 8 in 2024-02-07T12:24:29.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:29 smithi019 ceph-mon[29267]: pgmap v68: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:29.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:29 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:29.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:29 smithi019 ceph-mon[29267]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-02-07T12:24:29.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:29 smithi019 ceph-mon[29267]: osdmap e74: 8 total, 8 up, 8 in 2024-02-07T12:24:29.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:29 smithi172 ceph-mon[25457]: pgmap v68: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:29.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:29 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:29.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:29 smithi172 ceph-mon[25457]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-02-07T12:24:29.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:29 smithi172 ceph-mon[25457]: osdmap e74: 8 total, 8 up, 8 in 2024-02-07T12:24:31.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[29267]: osdmap e75: 8 total, 8 up, 8 in 2024-02-07T12:24:31.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:31.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:31.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:31.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:31.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[29267]: Saving service rgw.foo spec with placement count:2 2024-02-07T12:24:31.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:31.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi019.ciualv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:24:31.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi019.ciualv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:24:31.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi019.ciualv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-02-07T12:24:31.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:31.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:31.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[29267]: Deploying daemon rgw.foo.smithi019.ciualv on smithi019 2024-02-07T12:24:31.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[29267]: pgmap v71: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:31.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[24574]: osdmap e75: 8 total, 8 up, 8 in 2024-02-07T12:24:31.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:31.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:31.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:31.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:31.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[24574]: Saving service rgw.foo spec with placement count:2 2024-02-07T12:24:31.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:31.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi019.ciualv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:24:31.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi019.ciualv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:24:31.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi019.ciualv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-02-07T12:24:31.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:31.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:31.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[24574]: Deploying daemon rgw.foo.smithi019.ciualv on smithi019 2024-02-07T12:24:31.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:30 smithi019 ceph-mon[24574]: pgmap v71: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:31.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:30 smithi172 ceph-mon[25457]: osdmap e75: 8 total, 8 up, 8 in 2024-02-07T12:24:31.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:30 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:31.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:30 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:31.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:30 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:31.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:30 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:31.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:30 smithi172 ceph-mon[25457]: Saving service rgw.foo spec with placement count:2 2024-02-07T12:24:31.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:30 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:31.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:30 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi019.ciualv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:24:31.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:30 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi019.ciualv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:24:31.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:30 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi019.ciualv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-02-07T12:24:31.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:30 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:31.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:30 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:31.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:30 smithi172 ceph-mon[25457]: Deploying daemon rgw.foo.smithi019.ciualv on smithi019 2024-02-07T12:24:31.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:30 smithi172 ceph-mon[25457]: pgmap v71: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:31.720 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch apply iscsi foo u p' 2024-02-07T12:24:32.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:31 smithi019 ceph-mon[24574]: osdmap e76: 8 total, 8 up, 8 in 2024-02-07T12:24:32.090 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:31 smithi019 ceph-mon[29267]: osdmap e76: 8 total, 8 up, 8 in 2024-02-07T12:24:32.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:31 smithi172 ceph-mon[25457]: osdmap e76: 8 total, 8 up, 8 in 2024-02-07T12:24:33.281 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:32 smithi019 ceph-mon[24574]: pgmap v73: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 415 B/s wr, 0 op/s 2024-02-07T12:24:33.282 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:32 smithi019 ceph-mon[29267]: pgmap v73: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 415 B/s wr, 0 op/s 2024-02-07T12:24:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:32 smithi172 ceph-mon[25457]: pgmap v73: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 415 B/s wr, 0 op/s 2024-02-07T12:24:33.993 INFO:teuthology.orchestra.run.smithi019.stdout:Scheduled iscsi.foo update... 2024-02-07T12:24:34.899 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'sleep 180' 2024-02-07T12:24:35.068 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:34 smithi172 ceph-mon[25457]: from='client.14703 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:24:35.068 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:34 smithi172 ceph-mon[25457]: Saving service iscsi.foo spec with placement count:1 2024-02-07T12:24:35.068 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:34 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:35.068 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:34 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:35.068 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:34 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.wueguy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:24:35.068 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:34 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.wueguy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:24:35.068 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:34 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.wueguy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-02-07T12:24:35.068 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:34 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:35.068 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:34 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:35.068 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:34 smithi172 ceph-mon[25457]: Deploying daemon rgw.foo.smithi172.wueguy on smithi172 2024-02-07T12:24:35.069 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:34 smithi172 ceph-mon[25457]: pgmap v74: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 341 B/s wr, 0 op/s 2024-02-07T12:24:35.210 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:34 smithi019 ceph-mon[29267]: from='client.14703 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:24:35.210 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:34 smithi019 ceph-mon[29267]: Saving service iscsi.foo spec with placement count:1 2024-02-07T12:24:35.210 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:34 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:35.210 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:34 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:35.211 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:34 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.wueguy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:24:35.211 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:34 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.wueguy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:24:35.211 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:34 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.wueguy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-02-07T12:24:35.211 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:34 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:35.211 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:34 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:35.211 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:34 smithi019 ceph-mon[29267]: Deploying daemon rgw.foo.smithi172.wueguy on smithi172 2024-02-07T12:24:35.211 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:34 smithi019 ceph-mon[29267]: pgmap v74: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 341 B/s wr, 0 op/s 2024-02-07T12:24:35.211 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:34 smithi019 ceph-mon[24574]: from='client.14703 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:24:35.211 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:34 smithi019 ceph-mon[24574]: Saving service iscsi.foo spec with placement count:1 2024-02-07T12:24:35.211 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:34 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:35.211 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:34 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:35.212 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:34 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.wueguy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:24:35.212 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:34 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.wueguy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:24:35.212 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:34 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.wueguy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-02-07T12:24:35.212 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:34 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:35.212 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:34 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:35.212 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:34 smithi019 ceph-mon[24574]: Deploying daemon rgw.foo.smithi172.wueguy on smithi172 2024-02-07T12:24:35.212 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:34 smithi019 ceph-mon[24574]: pgmap v74: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 341 B/s wr, 0 op/s 2024-02-07T12:24:37.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:37 smithi019 ceph-mon[29267]: pgmap v75: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 270 B/s wr, 0 op/s 2024-02-07T12:24:37.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:37 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:37.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:37 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:24:37.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:37 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:37.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:37 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:24:37.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:37 smithi019 ceph-mon[24574]: pgmap v75: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 270 B/s wr, 0 op/s 2024-02-07T12:24:37.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:37 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:37.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:37 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:24:37.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:37 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:37.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:37 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:24:37.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:37 smithi172 ceph-mon[25457]: pgmap v75: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 270 B/s wr, 0 op/s 2024-02-07T12:24:37.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:37 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:37.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:37 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:24:37.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:37 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:37.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:37 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:24:38.298 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:24:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:24:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:24:38.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:24:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:24:38] "GET /metrics HTTP/1.1" 200 197910 "" "Prometheus/2.33.4" 2024-02-07T12:24:39.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:38 smithi019 ceph-mon[29267]: osdmap e77: 8 total, 8 up, 8 in 2024-02-07T12:24:39.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:38 smithi019 ceph-mon[29267]: pgmap v77: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 4.5 KiB/s wr, 265 op/s 2024-02-07T12:24:39.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:38 smithi019 ceph-mon[24574]: osdmap e77: 8 total, 8 up, 8 in 2024-02-07T12:24:39.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:38 smithi019 ceph-mon[24574]: pgmap v77: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 4.5 KiB/s wr, 265 op/s 2024-02-07T12:24:39.254 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:38 smithi172 ceph-mon[25457]: osdmap e77: 8 total, 8 up, 8 in 2024-02-07T12:24:39.254 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:38 smithi172 ceph-mon[25457]: pgmap v77: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 4.5 KiB/s wr, 265 op/s 2024-02-07T12:24:40.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:40 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:40.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:40 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:40.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:40 smithi019 ceph-mon[29267]: Checking dashboard <-> RGW credentials 2024-02-07T12:24:40.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:40 smithi019 ceph-mon[29267]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s rd, 3.8 KiB/s wr, 221 op/s 2024-02-07T12:24:40.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:40 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:40.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:40 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:40.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:40 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:40.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:40 smithi019 ceph-mon[29267]: Checking pool "foo" exists for service iscsi.foo 2024-02-07T12:24:40.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:40 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:24:40.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:40 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:24:40.989 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:40 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-02-07T12:24:40.989 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:40 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:40.989 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:40 smithi019 ceph-mon[29267]: Deploying daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:24:40.989 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:40 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:40.989 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:40 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:40.989 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:40 smithi019 ceph-mon[24574]: Checking dashboard <-> RGW credentials 2024-02-07T12:24:40.989 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:40 smithi019 ceph-mon[24574]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s rd, 3.8 KiB/s wr, 221 op/s 2024-02-07T12:24:40.989 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:40 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:40.989 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:40 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:40.989 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:40 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:40.989 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:40 smithi019 ceph-mon[24574]: Checking pool "foo" exists for service iscsi.foo 2024-02-07T12:24:40.990 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:40 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:24:40.990 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:40 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:24:40.990 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:40 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-02-07T12:24:40.990 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:40 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:40.990 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:40 smithi019 ceph-mon[24574]: Deploying daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:24:41.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:40 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:41.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:40 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:41.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:40 smithi172 ceph-mon[25457]: Checking dashboard <-> RGW credentials 2024-02-07T12:24:41.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:40 smithi172 ceph-mon[25457]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s rd, 3.8 KiB/s wr, 221 op/s 2024-02-07T12:24:41.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:40 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:41.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:40 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:41.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:40 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:41.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:40 smithi172 ceph-mon[25457]: Checking pool "foo" exists for service iscsi.foo 2024-02-07T12:24:41.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:40 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:24:41.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:40 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:24:41.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:40 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-02-07T12:24:41.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:40 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:41.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:40 smithi172 ceph-mon[25457]: Deploying daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:24:42.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:42 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:42.961 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:42 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:42.961 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:42 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:43.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:43 smithi019 ceph-mon[29267]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 115 KiB/s rd, 3.6 KiB/s wr, 216 op/s 2024-02-07T12:24:43.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:43 smithi019 ceph-mon[29267]: mgrmap e21: y(active, since 116s), standbys: x 2024-02-07T12:24:43.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:43 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:43.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:43 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:24:43.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:43 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:43.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:43 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:24:43.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:43 smithi019 ceph-mon[24574]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 115 KiB/s rd, 3.6 KiB/s wr, 216 op/s 2024-02-07T12:24:43.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:43 smithi019 ceph-mon[24574]: mgrmap e21: y(active, since 116s), standbys: x 2024-02-07T12:24:43.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:43 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:43.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:43 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:24:43.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:43 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:43.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:43 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:24:43.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:43 smithi172 ceph-mon[25457]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 115 KiB/s rd, 3.6 KiB/s wr, 216 op/s 2024-02-07T12:24:43.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:43 smithi172 ceph-mon[25457]: mgrmap e21: y(active, since 116s), standbys: x 2024-02-07T12:24:43.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:43 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:43.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:43 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:24:43.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:43 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:43.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:43 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:24:44.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:44 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/294165719' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-02-07T12:24:44.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:44 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3329572880' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3548359334"}]: dispatch 2024-02-07T12:24:44.857 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:44 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/294165719' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-02-07T12:24:44.857 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:44 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3329572880' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3548359334"}]: dispatch 2024-02-07T12:24:44.879 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:44 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/294165719' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-02-07T12:24:44.879 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:44 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3329572880' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3548359334"}]: dispatch 2024-02-07T12:24:45.824 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:45 smithi172 ceph-mon[25457]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 115 KiB/s rd, 3.6 KiB/s wr, 216 op/s 2024-02-07T12:24:45.824 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:45 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3329572880' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3548359334"}]': finished 2024-02-07T12:24:45.825 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:45 smithi172 ceph-mon[25457]: osdmap e78: 8 total, 8 up, 8 in 2024-02-07T12:24:45.825 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:45 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2421941694' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3958473332"}]: dispatch 2024-02-07T12:24:45.825 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:45 smithi172 ceph-mon[25457]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3958473332"}]: dispatch 2024-02-07T12:24:46.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:45 smithi019 ceph-mon[29267]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 115 KiB/s rd, 3.6 KiB/s wr, 216 op/s 2024-02-07T12:24:46.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:45 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3329572880' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3548359334"}]': finished 2024-02-07T12:24:46.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:45 smithi019 ceph-mon[29267]: osdmap e78: 8 total, 8 up, 8 in 2024-02-07T12:24:46.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:45 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2421941694' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3958473332"}]: dispatch 2024-02-07T12:24:46.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:45 smithi019 ceph-mon[29267]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3958473332"}]: dispatch 2024-02-07T12:24:46.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:45 smithi019 ceph-mon[24574]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 115 KiB/s rd, 3.6 KiB/s wr, 216 op/s 2024-02-07T12:24:46.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:45 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3329572880' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3548359334"}]': finished 2024-02-07T12:24:46.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:45 smithi019 ceph-mon[24574]: osdmap e78: 8 total, 8 up, 8 in 2024-02-07T12:24:46.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:45 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2421941694' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3958473332"}]: dispatch 2024-02-07T12:24:46.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:45 smithi019 ceph-mon[24574]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3958473332"}]: dispatch 2024-02-07T12:24:46.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[24574]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3958473332"}]': finished 2024-02-07T12:24:46.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[24574]: osdmap e79: 8 total, 8 up, 8 in 2024-02-07T12:24:46.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3603453757' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/840882240"}]: dispatch 2024-02-07T12:24:46.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]: dispatch 2024-02-07T12:24:46.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-02-07T12:24:46.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]: dispatch 2024-02-07T12:24:46.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-02-07T12:24:46.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-02-07T12:24:46.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-02-07T12:24:46.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-02-07T12:24:46.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-02-07T12:24:46.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:24:46.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:24:46.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:24:46.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:24:46.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:46.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[29267]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3958473332"}]': finished 2024-02-07T12:24:46.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[29267]: osdmap e79: 8 total, 8 up, 8 in 2024-02-07T12:24:46.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3603453757' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/840882240"}]: dispatch 2024-02-07T12:24:46.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]: dispatch 2024-02-07T12:24:46.847 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-02-07T12:24:46.847 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]: dispatch 2024-02-07T12:24:46.847 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-02-07T12:24:46.847 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-02-07T12:24:46.847 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-02-07T12:24:46.847 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-02-07T12:24:46.847 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-02-07T12:24:46.847 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:24:46.847 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:24:46.847 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:24:46.847 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:24:46.847 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:46 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:46 smithi172 ceph-mon[25457]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3958473332"}]': finished 2024-02-07T12:24:47.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:46 smithi172 ceph-mon[25457]: osdmap e79: 8 total, 8 up, 8 in 2024-02-07T12:24:47.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:46 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3603453757' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/840882240"}]: dispatch 2024-02-07T12:24:47.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:46 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]: dispatch 2024-02-07T12:24:47.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:46 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-02-07T12:24:47.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:46 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]: dispatch 2024-02-07T12:24:47.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:46 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-02-07T12:24:47.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:46 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-02-07T12:24:47.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:46 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-02-07T12:24:47.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:46 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-02-07T12:24:47.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:46 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-02-07T12:24:47.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:46 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:24:47.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:46 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:24:47.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:46 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:24:47.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:46 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:24:47.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:46 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:47.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:47 smithi172 ceph-mon[25457]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 5.2 KiB/s rd, 255 B/s wr, 5 op/s 2024-02-07T12:24:47.994 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:47 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3603453757' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/840882240"}]': finished 2024-02-07T12:24:47.994 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:47 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]': finished 2024-02-07T12:24:47.994 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:47 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-02-07T12:24:47.994 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:47 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-02-07T12:24:47.994 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:47 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-02-07T12:24:47.994 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:47 smithi172 ceph-mon[25457]: osdmap e80: 8 total, 8 up, 8 in 2024-02-07T12:24:47.994 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:47 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:47.994 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:47 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:47.994 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:47 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:47.994 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-02-07T12:24:47.994 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-02-07T12:24:47.994 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:47 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:47.994 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi019"}]: dispatch 2024-02-07T12:24:47.995 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:47 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:47.995 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:24:47.995 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:47.995 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:24:47.995 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:47 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2590212982' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2628806613"}]: dispatch 2024-02-07T12:24:48.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[24574]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 5.2 KiB/s rd, 255 B/s wr, 5 op/s 2024-02-07T12:24:48.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3603453757' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/840882240"}]': finished 2024-02-07T12:24:48.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]': finished 2024-02-07T12:24:48.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-02-07T12:24:48.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-02-07T12:24:48.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-02-07T12:24:48.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[24574]: osdmap e80: 8 total, 8 up, 8 in 2024-02-07T12:24:48.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:48.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:48.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:48.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-02-07T12:24:48.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-02-07T12:24:48.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:48.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi019"}]: dispatch 2024-02-07T12:24:48.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:48.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:24:48.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:48.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:24:48.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2590212982' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2628806613"}]: dispatch 2024-02-07T12:24:48.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[29267]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 5.2 KiB/s rd, 255 B/s wr, 5 op/s 2024-02-07T12:24:48.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3603453757' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/840882240"}]': finished 2024-02-07T12:24:48.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]': finished 2024-02-07T12:24:48.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-02-07T12:24:48.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-02-07T12:24:48.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-02-07T12:24:48.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[29267]: osdmap e80: 8 total, 8 up, 8 in 2024-02-07T12:24:48.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:48.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:48.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:48.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-02-07T12:24:48.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-02-07T12:24:48.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:48.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi019"}]: dispatch 2024-02-07T12:24:48.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:48.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:24:48.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:48.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:24:48.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:47 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2590212982' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2628806613"}]: dispatch 2024-02-07T12:24:48.371 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:24:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:24:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:24:48.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:24:48 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:24:48] "GET /metrics HTTP/1.1" 200 208090 "" "Prometheus/2.33.4" 2024-02-07T12:24:48.959 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:48 smithi019 ceph-mon[29267]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-02-07T12:24:48.959 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:48 smithi019 ceph-mon[29267]: Adding iSCSI gateway http://:@172.21.15.19:5000 to Dashboard 2024-02-07T12:24:48.959 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:48 smithi019 ceph-mon[29267]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-02-07T12:24:48.959 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:48 smithi019 ceph-mon[29267]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi019"}]: dispatch 2024-02-07T12:24:48.960 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:48 smithi019 ceph-mon[29267]: Checking dashboard <-> RGW credentials 2024-02-07T12:24:48.960 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:48 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2590212982' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2628806613"}]': finished 2024-02-07T12:24:48.960 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:48 smithi019 ceph-mon[29267]: osdmap e81: 8 total, 8 up, 8 in 2024-02-07T12:24:48.960 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:48 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:48.960 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:48 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1472811951' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/4244076805"}]: dispatch 2024-02-07T12:24:48.960 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:48 smithi019 ceph-mon[29267]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/4244076805"}]: dispatch 2024-02-07T12:24:48.960 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:48 smithi019 ceph-mon[29267]: pgmap v86: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:48.960 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:48 smithi019 ceph-mon[24574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-02-07T12:24:48.961 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:48 smithi019 ceph-mon[24574]: Adding iSCSI gateway http://:@172.21.15.19:5000 to Dashboard 2024-02-07T12:24:48.961 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:48 smithi019 ceph-mon[24574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-02-07T12:24:48.961 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:48 smithi019 ceph-mon[24574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi019"}]: dispatch 2024-02-07T12:24:48.961 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:48 smithi019 ceph-mon[24574]: Checking dashboard <-> RGW credentials 2024-02-07T12:24:48.961 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:48 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2590212982' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2628806613"}]': finished 2024-02-07T12:24:48.961 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:48 smithi019 ceph-mon[24574]: osdmap e81: 8 total, 8 up, 8 in 2024-02-07T12:24:48.961 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:48 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:48.961 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:48 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1472811951' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/4244076805"}]: dispatch 2024-02-07T12:24:48.961 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:48 smithi019 ceph-mon[24574]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/4244076805"}]: dispatch 2024-02-07T12:24:48.962 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:48 smithi019 ceph-mon[24574]: pgmap v86: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:49.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:48 smithi172 ceph-mon[25457]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-02-07T12:24:49.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:48 smithi172 ceph-mon[25457]: Adding iSCSI gateway http://:@172.21.15.19:5000 to Dashboard 2024-02-07T12:24:49.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:48 smithi172 ceph-mon[25457]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-02-07T12:24:49.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:48 smithi172 ceph-mon[25457]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi019"}]: dispatch 2024-02-07T12:24:49.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:48 smithi172 ceph-mon[25457]: Checking dashboard <-> RGW credentials 2024-02-07T12:24:49.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:48 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2590212982' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2628806613"}]': finished 2024-02-07T12:24:49.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:48 smithi172 ceph-mon[25457]: osdmap e81: 8 total, 8 up, 8 in 2024-02-07T12:24:49.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:48 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:24:49.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:48 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1472811951' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/4244076805"}]: dispatch 2024-02-07T12:24:49.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:48 smithi172 ceph-mon[25457]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/4244076805"}]: dispatch 2024-02-07T12:24:49.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:48 smithi172 ceph-mon[25457]: pgmap v86: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:50.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:49 smithi019 ceph-mon[24574]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/4244076805"}]': finished 2024-02-07T12:24:50.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:49 smithi019 ceph-mon[24574]: osdmap e82: 8 total, 8 up, 8 in 2024-02-07T12:24:50.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:49 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2333880324' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/4244076805"}]: dispatch 2024-02-07T12:24:50.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:49 smithi019 ceph-mon[24574]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/4244076805"}]: dispatch 2024-02-07T12:24:50.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:49 smithi019 ceph-mon[29267]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/4244076805"}]': finished 2024-02-07T12:24:50.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:49 smithi019 ceph-mon[29267]: osdmap e82: 8 total, 8 up, 8 in 2024-02-07T12:24:50.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:49 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2333880324' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/4244076805"}]: dispatch 2024-02-07T12:24:50.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:49 smithi019 ceph-mon[29267]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/4244076805"}]: dispatch 2024-02-07T12:24:50.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:49 smithi172 ceph-mon[25457]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/4244076805"}]': finished 2024-02-07T12:24:50.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:49 smithi172 ceph-mon[25457]: osdmap e82: 8 total, 8 up, 8 in 2024-02-07T12:24:50.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:49 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2333880324' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/4244076805"}]: dispatch 2024-02-07T12:24:50.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:49 smithi172 ceph-mon[25457]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/4244076805"}]: dispatch 2024-02-07T12:24:51.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:50 smithi019 ceph-mon[29267]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/4244076805"}]': finished 2024-02-07T12:24:51.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:50 smithi019 ceph-mon[29267]: osdmap e83: 8 total, 8 up, 8 in 2024-02-07T12:24:51.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:50 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1142662505' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/2471401560"}]: dispatch 2024-02-07T12:24:51.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:50 smithi019 ceph-mon[29267]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/2471401560"}]: dispatch 2024-02-07T12:24:51.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:50 smithi019 ceph-mon[29267]: pgmap v89: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:51.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:50 smithi019 ceph-mon[24574]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/4244076805"}]': finished 2024-02-07T12:24:51.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:50 smithi019 ceph-mon[24574]: osdmap e83: 8 total, 8 up, 8 in 2024-02-07T12:24:51.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:50 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1142662505' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/2471401560"}]: dispatch 2024-02-07T12:24:51.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:50 smithi019 ceph-mon[24574]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/2471401560"}]: dispatch 2024-02-07T12:24:51.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:50 smithi019 ceph-mon[24574]: pgmap v89: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:50 smithi172 ceph-mon[25457]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/4244076805"}]': finished 2024-02-07T12:24:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:50 smithi172 ceph-mon[25457]: osdmap e83: 8 total, 8 up, 8 in 2024-02-07T12:24:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:50 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1142662505' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/2471401560"}]: dispatch 2024-02-07T12:24:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:50 smithi172 ceph-mon[25457]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/2471401560"}]: dispatch 2024-02-07T12:24:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:50 smithi172 ceph-mon[25457]: pgmap v89: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:52.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:51 smithi172 ceph-mon[25457]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/2471401560"}]': finished 2024-02-07T12:24:52.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:51 smithi172 ceph-mon[25457]: osdmap e84: 8 total, 8 up, 8 in 2024-02-07T12:24:52.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:51 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1721941925' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/784284954"}]: dispatch 2024-02-07T12:24:52.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:51 smithi172 ceph-mon[25457]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/784284954"}]: dispatch 2024-02-07T12:24:52.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:51 smithi019 ceph-mon[29267]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/2471401560"}]': finished 2024-02-07T12:24:52.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:51 smithi019 ceph-mon[29267]: osdmap e84: 8 total, 8 up, 8 in 2024-02-07T12:24:52.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:51 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1721941925' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/784284954"}]: dispatch 2024-02-07T12:24:52.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:51 smithi019 ceph-mon[29267]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/784284954"}]: dispatch 2024-02-07T12:24:52.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:51 smithi019 ceph-mon[24574]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/2471401560"}]': finished 2024-02-07T12:24:52.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:51 smithi019 ceph-mon[24574]: osdmap e84: 8 total, 8 up, 8 in 2024-02-07T12:24:52.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:51 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1721941925' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/784284954"}]: dispatch 2024-02-07T12:24:52.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:51 smithi019 ceph-mon[24574]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/784284954"}]: dispatch 2024-02-07T12:24:53.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:52 smithi019 ceph-mon[24574]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/784284954"}]': finished 2024-02-07T12:24:53.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:52 smithi019 ceph-mon[24574]: osdmap e85: 8 total, 8 up, 8 in 2024-02-07T12:24:53.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:52 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1077573297' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/412909863"}]: dispatch 2024-02-07T12:24:53.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:52 smithi019 ceph-mon[24574]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 767 B/s wr, 12 op/s; 25 B/s, 0 objects/s recovering 2024-02-07T12:24:53.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:52 smithi019 ceph-mon[29267]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/784284954"}]': finished 2024-02-07T12:24:53.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:52 smithi019 ceph-mon[29267]: osdmap e85: 8 total, 8 up, 8 in 2024-02-07T12:24:53.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:52 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1077573297' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/412909863"}]: dispatch 2024-02-07T12:24:53.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:52 smithi019 ceph-mon[29267]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 767 B/s wr, 12 op/s; 25 B/s, 0 objects/s recovering 2024-02-07T12:24:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:52 smithi172 ceph-mon[25457]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/784284954"}]': finished 2024-02-07T12:24:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:52 smithi172 ceph-mon[25457]: osdmap e85: 8 total, 8 up, 8 in 2024-02-07T12:24:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:52 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1077573297' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/412909863"}]: dispatch 2024-02-07T12:24:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:52 smithi172 ceph-mon[25457]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 767 B/s wr, 12 op/s; 25 B/s, 0 objects/s recovering 2024-02-07T12:24:54.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:53 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1077573297' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/412909863"}]': finished 2024-02-07T12:24:54.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:53 smithi019 ceph-mon[24574]: osdmap e86: 8 total, 8 up, 8 in 2024-02-07T12:24:54.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:53 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1215090326' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/2571038478"}]: dispatch 2024-02-07T12:24:54.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:53 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:24:54.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:53 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1077573297' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/412909863"}]': finished 2024-02-07T12:24:54.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:53 smithi019 ceph-mon[29267]: osdmap e86: 8 total, 8 up, 8 in 2024-02-07T12:24:54.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:53 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1215090326' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/2571038478"}]: dispatch 2024-02-07T12:24:54.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:53 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:24:54.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:53 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1077573297' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/412909863"}]': finished 2024-02-07T12:24:54.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:53 smithi172 ceph-mon[25457]: osdmap e86: 8 total, 8 up, 8 in 2024-02-07T12:24:54.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:53 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1215090326' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/2571038478"}]: dispatch 2024-02-07T12:24:54.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:53 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:24:55.269 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:54 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1215090326' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/2571038478"}]': finished 2024-02-07T12:24:55.269 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:54 smithi019 ceph-mon[29267]: osdmap e87: 8 total, 8 up, 8 in 2024-02-07T12:24:55.269 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:54 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2399766536' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/2571038478"}]: dispatch 2024-02-07T12:24:55.270 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:54 smithi019 ceph-mon[29267]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 767 B/s wr, 12 op/s; 25 B/s, 0 objects/s recovering 2024-02-07T12:24:55.270 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:54 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1215090326' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/2571038478"}]': finished 2024-02-07T12:24:55.270 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:54 smithi019 ceph-mon[24574]: osdmap e87: 8 total, 8 up, 8 in 2024-02-07T12:24:55.270 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:54 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2399766536' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/2571038478"}]: dispatch 2024-02-07T12:24:55.270 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:54 smithi019 ceph-mon[24574]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 767 B/s wr, 12 op/s; 25 B/s, 0 objects/s recovering 2024-02-07T12:24:55.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:54 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1215090326' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/2571038478"}]': finished 2024-02-07T12:24:55.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:54 smithi172 ceph-mon[25457]: osdmap e87: 8 total, 8 up, 8 in 2024-02-07T12:24:55.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:54 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2399766536' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/2571038478"}]: dispatch 2024-02-07T12:24:55.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:54 smithi172 ceph-mon[25457]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 767 B/s wr, 12 op/s; 25 B/s, 0 objects/s recovering 2024-02-07T12:24:56.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:55 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2399766536' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/2571038478"}]': finished 2024-02-07T12:24:56.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:55 smithi019 ceph-mon[29267]: osdmap e88: 8 total, 8 up, 8 in 2024-02-07T12:24:56.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:55 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3804063431' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/2471401560"}]: dispatch 2024-02-07T12:24:56.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:55 smithi019 ceph-mon[29267]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/2471401560"}]: dispatch 2024-02-07T12:24:56.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:55 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2399766536' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/2571038478"}]': finished 2024-02-07T12:24:56.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:55 smithi019 ceph-mon[24574]: osdmap e88: 8 total, 8 up, 8 in 2024-02-07T12:24:56.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:55 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3804063431' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/2471401560"}]: dispatch 2024-02-07T12:24:56.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:55 smithi019 ceph-mon[24574]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/2471401560"}]: dispatch 2024-02-07T12:24:56.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:55 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2399766536' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/2571038478"}]': finished 2024-02-07T12:24:56.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:55 smithi172 ceph-mon[25457]: osdmap e88: 8 total, 8 up, 8 in 2024-02-07T12:24:56.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:55 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3804063431' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/2471401560"}]: dispatch 2024-02-07T12:24:56.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:55 smithi172 ceph-mon[25457]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/2471401560"}]: dispatch 2024-02-07T12:24:57.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:56 smithi019 ceph-mon[24574]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/2471401560"}]': finished 2024-02-07T12:24:57.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:56 smithi019 ceph-mon[24574]: osdmap e89: 8 total, 8 up, 8 in 2024-02-07T12:24:57.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:56 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3496779633' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1231318565"}]: dispatch 2024-02-07T12:24:57.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:56 smithi019 ceph-mon[24574]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:57.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:56 smithi019 ceph-mon[29267]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/2471401560"}]': finished 2024-02-07T12:24:57.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:56 smithi019 ceph-mon[29267]: osdmap e89: 8 total, 8 up, 8 in 2024-02-07T12:24:57.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:56 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3496779633' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1231318565"}]: dispatch 2024-02-07T12:24:57.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:56 smithi019 ceph-mon[29267]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:57.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:56 smithi172 ceph-mon[25457]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/2471401560"}]': finished 2024-02-07T12:24:57.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:56 smithi172 ceph-mon[25457]: osdmap e89: 8 total, 8 up, 8 in 2024-02-07T12:24:57.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:56 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3496779633' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1231318565"}]: dispatch 2024-02-07T12:24:57.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:56 smithi172 ceph-mon[25457]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:24:58.324 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:58 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3496779633' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1231318565"}]': finished 2024-02-07T12:24:58.324 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:58 smithi019 ceph-mon[24574]: osdmap e90: 8 total, 8 up, 8 in 2024-02-07T12:24:58.324 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:58 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/22703419' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2080165887"}]: dispatch 2024-02-07T12:24:58.324 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:58 smithi019 ceph-mon[24574]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2080165887"}]: dispatch 2024-02-07T12:24:58.325 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:58 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3496779633' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1231318565"}]': finished 2024-02-07T12:24:58.325 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:58 smithi019 ceph-mon[29267]: osdmap e90: 8 total, 8 up, 8 in 2024-02-07T12:24:58.325 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:58 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/22703419' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2080165887"}]: dispatch 2024-02-07T12:24:58.325 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:58 smithi019 ceph-mon[29267]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2080165887"}]: dispatch 2024-02-07T12:24:58.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:24:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:24:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:24:58.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:58 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3496779633' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1231318565"}]': finished 2024-02-07T12:24:58.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:58 smithi172 ceph-mon[25457]: osdmap e90: 8 total, 8 up, 8 in 2024-02-07T12:24:58.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:58 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/22703419' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2080165887"}]: dispatch 2024-02-07T12:24:58.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:58 smithi172 ceph-mon[25457]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2080165887"}]: dispatch 2024-02-07T12:24:58.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:24:58 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:24:58] "GET /metrics HTTP/1.1" 200 208090 "" "Prometheus/2.33.4" 2024-02-07T12:24:59.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:59 smithi019 ceph-mon[24574]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2080165887"}]': finished 2024-02-07T12:24:59.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:59 smithi019 ceph-mon[24574]: osdmap e91: 8 total, 8 up, 8 in 2024-02-07T12:24:59.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:59 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3991629602' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1819641709"}]: dispatch 2024-02-07T12:24:59.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:24:59 smithi019 ceph-mon[24574]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:24:59.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:59 smithi019 ceph-mon[29267]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2080165887"}]': finished 2024-02-07T12:24:59.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:59 smithi019 ceph-mon[29267]: osdmap e91: 8 total, 8 up, 8 in 2024-02-07T12:24:59.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:59 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3991629602' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1819641709"}]: dispatch 2024-02-07T12:24:59.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:24:59 smithi019 ceph-mon[29267]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:24:59.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:59 smithi172 ceph-mon[25457]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2080165887"}]': finished 2024-02-07T12:24:59.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:59 smithi172 ceph-mon[25457]: osdmap e91: 8 total, 8 up, 8 in 2024-02-07T12:24:59.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:59 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3991629602' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1819641709"}]: dispatch 2024-02-07T12:24:59.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:24:59 smithi172 ceph-mon[25457]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:00.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:00 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3991629602' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1819641709"}]': finished 2024-02-07T12:25:00.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:00 smithi019 ceph-mon[24574]: osdmap e92: 8 total, 8 up, 8 in 2024-02-07T12:25:00.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:00 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2523409371' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/4109166938"}]: dispatch 2024-02-07T12:25:00.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:00 smithi019 ceph-mon[24574]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/4109166938"}]: dispatch 2024-02-07T12:25:00.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:00 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3991629602' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1819641709"}]': finished 2024-02-07T12:25:00.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:00 smithi019 ceph-mon[29267]: osdmap e92: 8 total, 8 up, 8 in 2024-02-07T12:25:00.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:00 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2523409371' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/4109166938"}]: dispatch 2024-02-07T12:25:00.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:00 smithi019 ceph-mon[29267]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/4109166938"}]: dispatch 2024-02-07T12:25:00.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:00 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3991629602' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1819641709"}]': finished 2024-02-07T12:25:00.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:00 smithi172 ceph-mon[25457]: osdmap e92: 8 total, 8 up, 8 in 2024-02-07T12:25:00.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:00 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2523409371' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/4109166938"}]: dispatch 2024-02-07T12:25:00.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:00 smithi172 ceph-mon[25457]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/4109166938"}]: dispatch 2024-02-07T12:25:01.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:01 smithi019 ceph-mon[24574]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/4109166938"}]': finished 2024-02-07T12:25:01.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:01 smithi019 ceph-mon[24574]: osdmap e93: 8 total, 8 up, 8 in 2024-02-07T12:25:01.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:01 smithi019 ceph-mon[24574]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:01.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:01 smithi019 ceph-mon[29267]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/4109166938"}]': finished 2024-02-07T12:25:01.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:01 smithi019 ceph-mon[29267]: osdmap e93: 8 total, 8 up, 8 in 2024-02-07T12:25:01.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:01 smithi019 ceph-mon[29267]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:01.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:01 smithi172 ceph-mon[25457]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/4109166938"}]': finished 2024-02-07T12:25:01.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:01 smithi172 ceph-mon[25457]: osdmap e93: 8 total, 8 up, 8 in 2024-02-07T12:25:01.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:01 smithi172 ceph-mon[25457]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:03.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:03 smithi019 ceph-mon[24574]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-02-07T12:25:03.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:03 smithi019 ceph-mon[29267]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-02-07T12:25:03.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:03 smithi172 ceph-mon[25457]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-02-07T12:25:04.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:04 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:25:04.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:04 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:25:04.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:04 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:25:05.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:05 smithi019 ceph-mon[29267]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 784 B/s rd, 0 op/s 2024-02-07T12:25:05.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:05 smithi019 ceph-mon[24574]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 784 B/s rd, 0 op/s 2024-02-07T12:25:05.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:05 smithi172 ceph-mon[25457]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 784 B/s rd, 0 op/s 2024-02-07T12:25:07.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:07 smithi019 ceph-mon[29267]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-02-07T12:25:07.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:07 smithi019 ceph-mon[24574]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-02-07T12:25:07.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:07 smithi172 ceph-mon[25457]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-02-07T12:25:08.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:25:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:25:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:25:08.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:25:08 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:25:08] "GET /metrics HTTP/1.1" 200 208112 "" "Prometheus/2.33.4" 2024-02-07T12:25:09.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:08 smithi019 ceph-mon[29267]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:25:09.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:08 smithi019 ceph-mon[24574]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:25:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:08 smithi172 ceph-mon[25457]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:25:11.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:11 smithi019 ceph-mon[24574]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 973 B/s rd, 0 op/s 2024-02-07T12:25:11.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:11 smithi019 ceph-mon[29267]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 973 B/s rd, 0 op/s 2024-02-07T12:25:11.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:11 smithi172 ceph-mon[25457]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 973 B/s rd, 0 op/s 2024-02-07T12:25:13.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:13 smithi019 ceph-mon[29267]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:13.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:13 smithi019 ceph-mon[24574]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:13.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:13 smithi172 ceph-mon[25457]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:14.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:14 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:25:14.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:14 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:25:14.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:14 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:25:15.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:15 smithi019 ceph-mon[29267]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:15.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:15 smithi019 ceph-mon[24574]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:15.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:15 smithi172 ceph-mon[25457]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:17.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:17 smithi019 ceph-mon[24574]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:17.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:17 smithi019 ceph-mon[29267]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:17.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:17 smithi172 ceph-mon[25457]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:18.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:25:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:25:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:25:18.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:25:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:25:18] "GET /metrics HTTP/1.1" 200 208053 "" "Prometheus/2.33.4" 2024-02-07T12:25:19.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:18 smithi019 ceph-mon[24574]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:19.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:18 smithi019 ceph-mon[29267]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:19.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:18 smithi172 ceph-mon[25457]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:21.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:21 smithi019 ceph-mon[24574]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:21.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:21 smithi019 ceph-mon[29267]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:21.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:21 smithi172 ceph-mon[25457]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:23.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:23 smithi019 ceph-mon[24574]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:23.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:23 smithi019 ceph-mon[29267]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:23.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:23 smithi172 ceph-mon[25457]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:24.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:24 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:25:25.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:24 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:25:25.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:24 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:25:25.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:25 smithi172 ceph-mon[25457]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:26.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:25 smithi019 ceph-mon[29267]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:26.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:25 smithi019 ceph-mon[24574]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:27.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:27 smithi172 ceph-mon[25457]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:28.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:27 smithi019 ceph-mon[24574]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:28.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:27 smithi019 ceph-mon[29267]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:28.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:25:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:25:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:25:28.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:25:28 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:25:28] "GET /metrics HTTP/1.1" 200 208053 "" "Prometheus/2.33.4" 2024-02-07T12:25:29.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:28 smithi019 ceph-mon[24574]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:29.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:28 smithi019 ceph-mon[29267]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:28 smithi172 ceph-mon[25457]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:31.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:31 smithi019 ceph-mon[24574]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:31.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:31 smithi019 ceph-mon[29267]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:31.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:31 smithi172 ceph-mon[25457]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:33.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:33 smithi019 ceph-mon[24574]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:33.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:33 smithi019 ceph-mon[29267]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:33.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:33 smithi172 ceph-mon[25457]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:34.873 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:34 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:25:35.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:34 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:25:35.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:34 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:25:35.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:35 smithi172 ceph-mon[25457]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:36.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:35 smithi019 ceph-mon[24574]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:36.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:35 smithi019 ceph-mon[29267]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:37.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:37 smithi172 ceph-mon[25457]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:38.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:37 smithi019 ceph-mon[24574]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:38.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:37 smithi019 ceph-mon[29267]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:38.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:25:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:25:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:25:38.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:25:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:25:38] "GET /metrics HTTP/1.1" 200 207993 "" "Prometheus/2.33.4" 2024-02-07T12:25:39.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:38 smithi019 ceph-mon[24574]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:39.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:38 smithi019 ceph-mon[29267]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:39.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:38 smithi172 ceph-mon[25457]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:41.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:41 smithi019 ceph-mon[24574]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:41.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:41 smithi019 ceph-mon[29267]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:41.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:41 smithi172 ceph-mon[25457]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:43.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:43 smithi019 ceph-mon[24574]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:43.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:43 smithi019 ceph-mon[29267]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:43.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:43 smithi172 ceph-mon[25457]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:44.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:44 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:25:45.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:44 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:25:45.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:44 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:25:45.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:45 smithi172 ceph-mon[25457]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:46.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:45 smithi019 ceph-mon[29267]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:46.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:45 smithi019 ceph-mon[24574]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:46.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:46 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:25:46.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:46 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:25:47.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:46 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:25:47.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:46 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:25:47.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:46 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:25:47.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:46 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:25:47.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:47 smithi019 ceph-mon[29267]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:47.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:47 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:25:47.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:47 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:25:47.863 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:47 smithi019 ceph-mon[24574]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:47.863 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:47 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:25:47.863 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:47 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:25:47.863 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:47 smithi172 ceph-mon[25457]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:47.863 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:47 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:25:47.863 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:47 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:25:48.119 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:25:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:25:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:25:48.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:25:48 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:25:48] "GET /metrics HTTP/1.1" 200 208008 "" "Prometheus/2.33.4" 2024-02-07T12:25:48.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:48 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:25:48.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:48 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:25:48.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:48 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:25:48.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:48 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:25:49.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:48 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:25:49.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:48 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:25:49.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:48 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:25:49.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:48 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:25:49.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:48 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:25:49.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:48 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:25:49.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:48 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:25:49.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:48 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:25:49.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:49 smithi172 ceph-mon[25457]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:50.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:49 smithi019 ceph-mon[24574]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:50.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:49 smithi019 ceph-mon[29267]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:52.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:51 smithi019 ceph-mon[24574]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:52.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:51 smithi019 ceph-mon[29267]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:52.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:51 smithi172 ceph-mon[25457]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:54.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:53 smithi019 ceph-mon[24574]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:54.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:53 smithi019 ceph-mon[29267]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:54.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:53 smithi172 ceph-mon[25457]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:55.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:54 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:25:55.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:54 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:25:55.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:54 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:25:56.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:55 smithi019 ceph-mon[24574]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:56.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:55 smithi019 ceph-mon[29267]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:56.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:55 smithi172 ceph-mon[25457]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:57.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:57 smithi172 ceph-mon[25457]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:58.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:57 smithi019 ceph-mon[24574]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:58.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:57 smithi019 ceph-mon[29267]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:25:58.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:25:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:25:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:25:58.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:25:58 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:25:58] "GET /metrics HTTP/1.1" 200 208008 "" "Prometheus/2.33.4" 2024-02-07T12:25:59.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:25:58 smithi019 ceph-mon[29267]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:59.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:25:58 smithi019 ceph-mon[24574]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:25:59.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:25:58 smithi172 ceph-mon[25457]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:01.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:01 smithi172 ceph-mon[25457]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:02.048 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:01 smithi019 ceph-mon[29267]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:02.048 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:01 smithi019 ceph-mon[24574]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:03.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:03 smithi172 ceph-mon[25457]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:04.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:03 smithi019 ceph-mon[24574]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:04.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:03 smithi019 ceph-mon[29267]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:04.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:04 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:26:05.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:04 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:26:05.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:04 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:26:05.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:05 smithi172 ceph-mon[25457]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:06.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:05 smithi019 ceph-mon[24574]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:06.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:05 smithi019 ceph-mon[29267]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:07.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:07 smithi172 ceph-mon[25457]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:08.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:07 smithi019 ceph-mon[24574]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:08.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:07 smithi019 ceph-mon[29267]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:08.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:26:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:26:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:26:08.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:26:08 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:26:08] "GET /metrics HTTP/1.1" 200 208014 "" "Prometheus/2.33.4" 2024-02-07T12:26:09.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:08 smithi019 ceph-mon[29267]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:09.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:08 smithi019 ceph-mon[24574]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:08 smithi172 ceph-mon[25457]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:11.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:11 smithi172 ceph-mon[25457]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:12.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:11 smithi019 ceph-mon[29267]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:12.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:11 smithi019 ceph-mon[24574]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:13.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:13 smithi172 ceph-mon[25457]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:14.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:13 smithi019 ceph-mon[29267]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:14.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:13 smithi019 ceph-mon[24574]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:14.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:14 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:26:15.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:14 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:26:15.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:14 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:26:15.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:15 smithi172 ceph-mon[25457]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:16.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:15 smithi019 ceph-mon[29267]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:16.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:15 smithi019 ceph-mon[24574]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:17.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:17 smithi172 ceph-mon[25457]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:18.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:17 smithi019 ceph-mon[29267]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:18.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:17 smithi019 ceph-mon[24574]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:18.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:26:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:26:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:26:18.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:26:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:26:18] "GET /metrics HTTP/1.1" 200 208013 "" "Prometheus/2.33.4" 2024-02-07T12:26:19.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:18 smithi019 ceph-mon[29267]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:19.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:18 smithi019 ceph-mon[24574]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:19.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:18 smithi172 ceph-mon[25457]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:21.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:21 smithi172 ceph-mon[25457]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:22.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:21 smithi019 ceph-mon[29267]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:22.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:21 smithi019 ceph-mon[24574]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:23.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:23 smithi172 ceph-mon[25457]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:24.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:23 smithi019 ceph-mon[29267]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:24.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:23 smithi019 ceph-mon[24574]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:24.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:24 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:26:25.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:24 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:26:25.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:24 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:26:25.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:25 smithi172 ceph-mon[25457]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:26.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:25 smithi019 ceph-mon[29267]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:26.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:25 smithi019 ceph-mon[24574]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:27.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:27 smithi172 ceph-mon[25457]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:28.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:27 smithi019 ceph-mon[24574]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:28.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:27 smithi019 ceph-mon[29267]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:28.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:26:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:26:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:26:28.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:26:28 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:26:28] "GET /metrics HTTP/1.1" 200 208013 "" "Prometheus/2.33.4" 2024-02-07T12:26:29.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:28 smithi019 ceph-mon[29267]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:29.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:28 smithi019 ceph-mon[24574]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:28 smithi172 ceph-mon[25457]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:31.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:31 smithi172 ceph-mon[25457]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:32.047 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:31 smithi019 ceph-mon[29267]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:32.048 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:31 smithi019 ceph-mon[24574]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:33.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:33 smithi172 ceph-mon[25457]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:34.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:33 smithi019 ceph-mon[24574]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:34.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:33 smithi019 ceph-mon[29267]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:34.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:34 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:26:35.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:34 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:26:35.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:34 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:26:35.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:35 smithi172 ceph-mon[25457]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:36.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:35 smithi019 ceph-mon[24574]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:36.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:35 smithi019 ceph-mon[29267]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:37.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:37 smithi172 ceph-mon[25457]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:38.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:37 smithi019 ceph-mon[29267]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:38.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:37 smithi019 ceph-mon[24574]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:38.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:26:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:26:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:26:38.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:26:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:26:38] "GET /metrics HTTP/1.1" 200 208022 "" "Prometheus/2.33.4" 2024-02-07T12:26:39.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:38 smithi019 ceph-mon[24574]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:39.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:38 smithi019 ceph-mon[29267]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:39.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:38 smithi172 ceph-mon[25457]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:41.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:41 smithi172 ceph-mon[25457]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:42.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:41 smithi019 ceph-mon[24574]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:42.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:41 smithi019 ceph-mon[29267]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:43.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:43 smithi172 ceph-mon[25457]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:44.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:43 smithi019 ceph-mon[24574]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:44.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:43 smithi019 ceph-mon[29267]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:44.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:44 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:26:45.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:44 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:26:45.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:44 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:26:45.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:45 smithi172 ceph-mon[25457]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:46.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:45 smithi019 ceph-mon[24574]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:46.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:45 smithi019 ceph-mon[29267]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:47.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:46 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:26:47.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:46 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:26:47.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:46 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:26:47.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:46 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:26:47.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:46 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:26:47.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:46 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:26:47.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:46 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:26:47.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:46 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:26:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:46 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:26:47.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:46 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:26:47.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:46 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:26:47.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:46 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:26:47.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:47 smithi172 ceph-mon[25457]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:48.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:47 smithi019 ceph-mon[24574]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:48.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:47 smithi019 ceph-mon[29267]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:48.263 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:26:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:26:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:26:48.512 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:26:48 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:26:48] "GET /metrics HTTP/1.1" 200 208011 "" "Prometheus/2.33.4" 2024-02-07T12:26:49.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:48 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:26:49.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:48 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:26:49.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:48 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:26:49.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:48 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:26:49.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:48 smithi019 ceph-mon[29267]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:49.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:48 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:26:49.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:48 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:26:49.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:48 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:26:49.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:48 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:26:49.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:48 smithi019 ceph-mon[24574]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:49.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:48 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:26:49.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:48 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:26:49.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:48 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:26:49.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:48 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:26:49.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:48 smithi172 ceph-mon[25457]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:51.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:51 smithi172 ceph-mon[25457]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:52.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:51 smithi019 ceph-mon[29267]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:52.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:51 smithi019 ceph-mon[24574]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:53.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:53 smithi172 ceph-mon[25457]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:54.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:53 smithi019 ceph-mon[29267]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:54.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:53 smithi019 ceph-mon[24574]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:55.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:55 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:26:55.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:55 smithi172 ceph-mon[25457]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:56.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:55 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:26:56.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:55 smithi019 ceph-mon[24574]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:56.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:55 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:26:56.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:55 smithi019 ceph-mon[29267]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:57.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:57 smithi172 ceph-mon[25457]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:58.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:57 smithi019 ceph-mon[24574]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:58.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:57 smithi019 ceph-mon[29267]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:26:58.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:26:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:26:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:26:58.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:26:58 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:26:58] "GET /metrics HTTP/1.1" 200 208011 "" "Prometheus/2.33.4" 2024-02-07T12:26:59.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:26:58 smithi019 ceph-mon[29267]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:59.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:26:58 smithi019 ceph-mon[24574]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:26:59.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:26:58 smithi172 ceph-mon[25457]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:01.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:01 smithi172 ceph-mon[25457]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:02.047 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:01 smithi019 ceph-mon[29267]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:02.048 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:01 smithi019 ceph-mon[24574]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:03.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:03 smithi172 ceph-mon[25457]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:04.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:03 smithi019 ceph-mon[29267]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:04.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:03 smithi019 ceph-mon[24574]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:06.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:05 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:27:06.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:05 smithi019 ceph-mon[29267]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:06.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:05 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:27:06.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:05 smithi019 ceph-mon[24574]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:06.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:05 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:27:06.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:05 smithi172 ceph-mon[25457]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:07.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:07 smithi172 ceph-mon[25457]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:08.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:07 smithi019 ceph-mon[29267]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:08.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:07 smithi019 ceph-mon[24574]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:08.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:27:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:27:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:27:08.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:27:08 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:27:08] "GET /metrics HTTP/1.1" 200 208002 "" "Prometheus/2.33.4" 2024-02-07T12:27:09.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:08 smithi019 ceph-mon[29267]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:09.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:08 smithi019 ceph-mon[24574]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:08 smithi172 ceph-mon[25457]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:11.872 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:11 smithi172 ceph-mon[25457]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:12.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:11 smithi019 ceph-mon[29267]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:12.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:11 smithi019 ceph-mon[24574]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:14.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:13 smithi019 ceph-mon[29267]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:14.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:13 smithi019 ceph-mon[24574]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:14.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:13 smithi172 ceph-mon[25457]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:16.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:15 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:27:16.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:15 smithi019 ceph-mon[24574]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:16.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:15 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:27:16.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:15 smithi019 ceph-mon[29267]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:16.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:15 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:27:16.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:15 smithi172 ceph-mon[25457]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:17.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:17 smithi172 ceph-mon[25457]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:18.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:17 smithi019 ceph-mon[24574]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:18.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:17 smithi019 ceph-mon[29267]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:18.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:27:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:27:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:27:18.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:27:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:27:18] "GET /metrics HTTP/1.1" 200 207971 "" "Prometheus/2.33.4" 2024-02-07T12:27:19.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:18 smithi019 ceph-mon[29267]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:19.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:18 smithi019 ceph-mon[24574]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:19.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:18 smithi172 ceph-mon[25457]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:22.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:21 smithi019 ceph-mon[29267]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:22.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:21 smithi019 ceph-mon[24574]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:22.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:21 smithi172 ceph-mon[25457]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:24.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:23 smithi019 ceph-mon[24574]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:24.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:23 smithi019 ceph-mon[29267]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:24.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:23 smithi172 ceph-mon[25457]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:26.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:25 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:27:26.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:25 smithi019 ceph-mon[24574]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:26.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:25 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:27:26.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:25 smithi019 ceph-mon[29267]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:26.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:25 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:27:26.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:25 smithi172 ceph-mon[25457]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:27.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:27 smithi172 ceph-mon[25457]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:28.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:27 smithi019 ceph-mon[24574]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:28.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:27 smithi019 ceph-mon[29267]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:28.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:27:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:27:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:27:28.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:27:28 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:27:28] "GET /metrics HTTP/1.1" 200 207971 "" "Prometheus/2.33.4" 2024-02-07T12:27:29.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:28 smithi019 ceph-mon[24574]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:29.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:28 smithi019 ceph-mon[29267]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:28 smithi172 ceph-mon[25457]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:32.047 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:31 smithi019 ceph-mon[29267]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:32.047 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:31 smithi019 ceph-mon[24574]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:32.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:31 smithi172 ceph-mon[25457]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:33.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:33 smithi019 ceph-mon[29267]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:33.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:33 smithi019 ceph-mon[24574]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:34.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:33 smithi172 ceph-mon[25457]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:36.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:35 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:27:36.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:35 smithi019 ceph-mon[29267]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:36.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:35 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:27:36.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:35 smithi019 ceph-mon[24574]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:36.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:35 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:27:36.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:35 smithi172 ceph-mon[25457]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:37.472 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-02-07T12:27:37.784 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:37 smithi019 ceph-mon[29267]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:37.785 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:37 smithi019 ceph-mon[24574]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:37.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:37 smithi172 ceph-mon[25457]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:38.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:27:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:27:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:27:38.448 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:27:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:27:38] "GET /metrics HTTP/1.1" 200 207962 "" "Prometheus/2.33.4" 2024-02-07T12:27:39.287 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:38 smithi019 ceph-mon[29267]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:39.288 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:38 smithi019 ceph-mon[24574]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:39.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:38 smithi172 ceph-mon[25457]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:40.083 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-02-07T12:27:42.072 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:41 smithi172 ceph-mon[25457]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:42.081 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:41 smithi019 ceph-mon[24574]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:42.081 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:41 smithi019 ceph-mon[29267]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:42.680 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph config set global log_to_journald false --force' 2024-02-07T12:27:43.976 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:43 smithi019 ceph-mon[29267]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:43.976 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:43 smithi019 ceph-mon[24574]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:44.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:43 smithi172 ceph-mon[25457]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:45.064 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch ps' 2024-02-07T12:27:46.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:45 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:27:46.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:45 smithi019 ceph-mon[24574]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:46.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:45 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:27:46.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:45 smithi019 ceph-mon[29267]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:46.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:45 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:27:46.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:45 smithi172 ceph-mon[25457]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:46.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:46 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:27:46.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:46 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:27:46.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:46 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:27:46.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:46 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:27:46.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:46 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:27:46.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:46 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:27:46.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:46 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:27:46.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:46 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:27:46.984 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:27:46.984 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (3m) 3m ago 3m 15.4M - ba2b418f427c 4eb1830b3c20 2024-02-07T12:27:46.984 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (3m) 3m ago 3m 48.8M - 8.3.5 dad864ee21e9 fcafad07f161 2024-02-07T12:27:46.984 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 running (3m) 3m ago 3m 73.4M - 3.5 e1d6a67b021e 32cb504d4898 2024-02-07T12:27:46.985 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443 running (8m) 3m ago 8m 411M - 17.2.0 e1d6a67b021e 3ce4fcd95143 2024-02-07T12:27:46.985 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:9283 running (10m) 3m ago 10m 454M - 17.2.0 e1d6a67b021e c025f688a668 2024-02-07T12:27:46.985 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (10m) 3m ago 10m 46.5M 2048M 17.2.0 e1d6a67b021e f3a8a050cf1d 2024-02-07T12:27:46.985 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (8m) 3m ago 8m 38.9M 2048M 17.2.0 e1d6a67b021e 7d62c7c4d075 2024-02-07T12:27:46.985 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (8m) 3m ago 8m 42.3M 2048M 17.2.0 e1d6a67b021e b9825d0db32d 2024-02-07T12:27:46.985 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (4m) 3m ago 4m 17.3M - 1dbe0e931976 de9599dbfeff 2024-02-07T12:27:46.985 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (4m) 3m ago 4m 15.9M - 1dbe0e931976 30f1dfd23ff2 2024-02-07T12:27:46.985 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (8m) 3m ago 8m 47.2M 4017M 17.2.0 e1d6a67b021e 87f2560a9b15 2024-02-07T12:27:46.985 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (7m) 3m ago 7m 50.6M 4017M 17.2.0 e1d6a67b021e 0f9a190f0e0d 2024-02-07T12:27:46.985 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (7m) 3m ago 7m 46.8M 4017M 17.2.0 e1d6a67b021e 7db94f25a1aa 2024-02-07T12:27:46.985 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (6m) 3m ago 6m 45.3M 4017M 17.2.0 e1d6a67b021e 0b6b674188ed 2024-02-07T12:27:46.985 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (6m) 3m ago 6m 45.3M 4272M 17.2.0 e1d6a67b021e df72cfd69b3b 2024-02-07T12:27:46.985 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (6m) 3m ago 6m 44.2M 4272M 17.2.0 e1d6a67b021e 3bdb22c07537 2024-02-07T12:27:46.986 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (5m) 3m ago 5m 43.1M 4272M 17.2.0 e1d6a67b021e 7859b651e0d9 2024-02-07T12:27:46.986 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (5m) 3m ago 5m 47.1M 4272M 17.2.0 e1d6a67b021e b301e7c75d73 2024-02-07T12:27:46.986 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (3m) 3m ago 4m 44.8M - 514e6a882f6e d482181f6c26 2024-02-07T12:27:46.986 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (3m) 3m ago 3m 71.2M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:27:46.986 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (3m) 3m ago 3m 70.5M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:27:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:46 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:27:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:46 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:27:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:46 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:27:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:46 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:27:47.586 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph versions' 2024-02-07T12:27:47.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:47 smithi019 ceph-mon[29267]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:47.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:47 smithi019 ceph-mon[24574]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:47.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:47 smithi172 ceph-mon[25457]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:48.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:27:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:27:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:27:48.545 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:27:48 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:27:48] "GET /metrics HTTP/1.1" 200 207976 "" "Prometheus/2.33.4" 2024-02-07T12:27:48.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:48 smithi019 ceph-mon[29267]: from='client.24856 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:27:48.837 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:48 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:27:48.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:48 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:27:48.838 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:48 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:27:48.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:48 smithi019 ceph-mon[24574]: from='client.24856 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:27:48.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:48 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:27:48.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:48 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:27:48.838 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:48 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:27:48.867 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:48 smithi172 ceph-mon[25457]: from='client.24856 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:27:48.868 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:48 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:27:48.868 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:48 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:27:48.868 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:48 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:27:49.538 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:27:49.538 INFO:teuthology.orchestra.run.smithi019.stdout: "mon": { 2024-02-07T12:27:49.538 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-02-07T12:27:49.538 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:27:49.538 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr": { 2024-02-07T12:27:49.538 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:27:49.538 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:27:49.538 INFO:teuthology.orchestra.run.smithi019.stdout: "osd": { 2024-02-07T12:27:49.538 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-02-07T12:27:49.538 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:27:49.538 INFO:teuthology.orchestra.run.smithi019.stdout: "mds": {}, 2024-02-07T12:27:49.539 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw": { 2024-02-07T12:27:49.539 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:27:49.539 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:27:49.539 INFO:teuthology.orchestra.run.smithi019.stdout: "overall": { 2024-02-07T12:27:49.539 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 15 2024-02-07T12:27:49.539 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:27:49.539 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:27:49.819 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:49 smithi019 ceph-mon[29267]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:49.819 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:49 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:27:49.819 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:49 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3084628499' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:27:49.819 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:49 smithi019 ceph-mon[24574]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:49.819 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:49 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:27:49.820 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:49 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3084628499' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:27:50.067 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph -s' 2024-02-07T12:27:50.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:49 smithi172 ceph-mon[25457]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:50.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:49 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:27:50.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:49 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3084628499' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:27:52.081 INFO:teuthology.orchestra.run.smithi019.stdout: cluster: 2024-02-07T12:27:52.081 INFO:teuthology.orchestra.run.smithi019.stdout: id: ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:27:52.081 INFO:teuthology.orchestra.run.smithi019.stdout: health: HEALTH_OK 2024-02-07T12:27:52.081 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:27:52.081 INFO:teuthology.orchestra.run.smithi019.stdout: services: 2024-02-07T12:27:52.081 INFO:teuthology.orchestra.run.smithi019.stdout: mon: 3 daemons, quorum a,c,b (age 8m) 2024-02-07T12:27:52.081 INFO:teuthology.orchestra.run.smithi019.stdout: mgr: y(active, since 5m), standbys: x 2024-02-07T12:27:52.081 INFO:teuthology.orchestra.run.smithi019.stdout: osd: 8 osds: 8 up (since 5m), 8 in (since 5m) 2024-02-07T12:27:52.081 INFO:teuthology.orchestra.run.smithi019.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-02-07T12:27:52.081 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:27:52.081 INFO:teuthology.orchestra.run.smithi019.stdout: data: 2024-02-07T12:27:52.082 INFO:teuthology.orchestra.run.smithi019.stdout: pools: 6 pools, 161 pgs 2024-02-07T12:27:52.082 INFO:teuthology.orchestra.run.smithi019.stdout: objects: 209 objects, 585 KiB 2024-02-07T12:27:52.082 INFO:teuthology.orchestra.run.smithi019.stdout: usage: 63 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:27:52.082 INFO:teuthology.orchestra.run.smithi019.stdout: pgs: 161 active+clean 2024-02-07T12:27:52.082 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:27:52.082 INFO:teuthology.orchestra.run.smithi019.stdout: io: 2024-02-07T12:27:52.082 INFO:teuthology.orchestra.run.smithi019.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-02-07T12:27:52.082 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:27:52.082 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:51 smithi019 ceph-mon[24574]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:52.083 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:51 smithi019 ceph-mon[29267]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:52.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:51 smithi172 ceph-mon[25457]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:52.758 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:52 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1583211570' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-02-07T12:27:52.758 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:52 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1583211570' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-02-07T12:27:52.759 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch ls' 2024-02-07T12:27:53.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:52 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1583211570' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-02-07T12:27:53.948 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:53 smithi019 ceph-mon[24574]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:53.948 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:53 smithi019 ceph-mon[29267]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:54.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:53 smithi172 ceph-mon[25457]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:54.579 INFO:teuthology.orchestra.run.smithi019.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-02-07T12:27:54.580 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager ?:9093,9094 1/1 3m ago 5m smithi019=a;count:1 2024-02-07T12:27:54.580 INFO:teuthology.orchestra.run.smithi019.stdout:grafana ?:3000 1/1 3m ago 5m smithi172=a;count:1 2024-02-07T12:27:54.580 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo 1/1 3m ago 3m count:1 2024-02-07T12:27:54.580 INFO:teuthology.orchestra.run.smithi019.stdout:mgr 2/2 3m ago 8m smithi019=y;smithi172=x;count:2 2024-02-07T12:27:54.580 INFO:teuthology.orchestra.run.smithi019.stdout:mon 3/3 3m ago 8m smithi019:172.21.15.19=a;smithi019:[v2:172.21.15.19:3301,v1:172.21.15.19:6790]=c;smithi172:172.21.15.172=b;count:3 2024-02-07T12:27:54.580 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter ?:9100 2/2 3m ago 5m smithi019=a;smithi172=b;count:2 2024-02-07T12:27:54.580 INFO:teuthology.orchestra.run.smithi019.stdout:osd 8 3m ago - 2024-02-07T12:27:54.580 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus ?:9095 1/1 3m ago 5m smithi172=a;count:1 2024-02-07T12:27:54.581 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo ?:8000 2/2 3m ago 3m count:2 2024-02-07T12:27:55.178 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-02-07T12:27:55.737 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:55 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:27:55.738 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:55 smithi019 ceph-mon[24574]: from='client.24874 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:27:55.738 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:55 smithi019 ceph-mon[24574]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:55.738 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:55 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:27:55.738 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:55 smithi019 ceph-mon[29267]: from='client.24874 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:27:55.738 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:55 smithi019 ceph-mon[29267]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:56.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:55 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:27:56.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:55 smithi172 ceph-mon[25457]: from='client.24874 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:27:56.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:55 smithi172 ceph-mon[25457]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:57.584 INFO:teuthology.orchestra.run.smithi019.stdout:Scheduled to redeploy mgr.x on host 'smithi172' 2024-02-07T12:27:57.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:57 smithi019 ceph-mon[29267]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:57.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:57 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3630844118' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-02-07T12:27:57.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:57 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:27:57.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:57 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:27:57.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:57 smithi019 ceph-mon[24574]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:57.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:57 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3630844118' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-02-07T12:27:57.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:57 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:27:57.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:57 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:27:57.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:57 smithi172 ceph-mon[25457]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:27:57.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:57 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3630844118' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-02-07T12:27:57.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:57 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:27:57.994 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:57 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:27:58.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:27:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:27:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:27:58.405 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch ps --refresh' 2024-02-07T12:27:58.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:27:58 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:27:58] "GET /metrics HTTP/1.1" 200 207976 "" "Prometheus/2.33.4" 2024-02-07T12:27:58.908 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:58 smithi019 ceph-mon[29267]: from='client.15000 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:27:58.908 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:58 smithi019 ceph-mon[29267]: Schedule redeploy daemon mgr.x 2024-02-07T12:27:58.908 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:58 smithi019 ceph-mon[24574]: from='client.15000 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:27:58.908 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:58 smithi019 ceph-mon[24574]: Schedule redeploy daemon mgr.x 2024-02-07T12:27:59.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:58 smithi172 ceph-mon[25457]: from='client.15000 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:27:59.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:58 smithi172 ceph-mon[25457]: Schedule redeploy daemon mgr.x 2024-02-07T12:27:59.909 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:27:59 smithi019 ceph-mon[29267]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:27:59.909 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:27:59 smithi019 ceph-mon[24574]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:00.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:27:59 smithi172 ceph-mon[25457]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:00.549 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:28:00.549 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (3m) 3m ago 4m 15.4M - ba2b418f427c 4eb1830b3c20 2024-02-07T12:28:00.549 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (3m) 3m ago 3m 48.8M - 8.3.5 dad864ee21e9 fcafad07f161 2024-02-07T12:28:00.550 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 running (3m) 3m ago 3m 73.4M - 3.5 e1d6a67b021e 32cb504d4898 2024-02-07T12:28:00.550 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443 running (8m) 3m ago 8m 411M - 17.2.0 e1d6a67b021e 3ce4fcd95143 2024-02-07T12:28:00.550 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:9283 running (10m) 3m ago 10m 454M - 17.2.0 e1d6a67b021e c025f688a668 2024-02-07T12:28:00.550 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (10m) 3m ago 10m 46.5M 2048M 17.2.0 e1d6a67b021e f3a8a050cf1d 2024-02-07T12:28:00.550 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (8m) 3m ago 8m 38.9M 2048M 17.2.0 e1d6a67b021e 7d62c7c4d075 2024-02-07T12:28:00.550 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (9m) 3m ago 9m 42.3M 2048M 17.2.0 e1d6a67b021e b9825d0db32d 2024-02-07T12:28:00.550 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (4m) 3m ago 4m 17.3M - 1dbe0e931976 de9599dbfeff 2024-02-07T12:28:00.550 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (4m) 3m ago 4m 15.9M - 1dbe0e931976 30f1dfd23ff2 2024-02-07T12:28:00.550 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (8m) 3m ago 8m 47.2M 4017M 17.2.0 e1d6a67b021e 87f2560a9b15 2024-02-07T12:28:00.550 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (7m) 3m ago 7m 50.6M 4017M 17.2.0 e1d6a67b021e 0f9a190f0e0d 2024-02-07T12:28:00.550 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (7m) 3m ago 7m 46.8M 4017M 17.2.0 e1d6a67b021e 7db94f25a1aa 2024-02-07T12:28:00.550 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (7m) 3m ago 7m 45.3M 4017M 17.2.0 e1d6a67b021e 0b6b674188ed 2024-02-07T12:28:00.550 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (6m) 3m ago 6m 45.3M 4272M 17.2.0 e1d6a67b021e df72cfd69b3b 2024-02-07T12:28:00.550 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (6m) 3m ago 6m 44.2M 4272M 17.2.0 e1d6a67b021e 3bdb22c07537 2024-02-07T12:28:00.551 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (6m) 3m ago 6m 43.1M 4272M 17.2.0 e1d6a67b021e 7859b651e0d9 2024-02-07T12:28:00.551 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (5m) 3m ago 5m 47.1M 4272M 17.2.0 e1d6a67b021e b301e7c75d73 2024-02-07T12:28:00.551 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (3m) 3m ago 4m 44.8M - 514e6a882f6e d482181f6c26 2024-02-07T12:28:00.551 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (3m) 3m ago 3m 71.2M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:28:00.551 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (3m) 3m ago 3m 70.5M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:28:01.190 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'sleep 180' 2024-02-07T12:28:01.817 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:01 smithi019 ceph-mon[24574]: from='client.24892 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:28:01.817 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:01 smithi019 ceph-mon[24574]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:01.817 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:01 smithi019 ceph-mon[29267]: from='client.24892 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:28:01.817 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:01 smithi019 ceph-mon[29267]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:02.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:01 smithi172 ceph-mon[25457]: from='client.24892 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:28:02.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:01 smithi172 ceph-mon[25457]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:04.084 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:03 smithi019 ceph-mon[24574]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:04.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:03 smithi019 ceph-mon[29267]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:04.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:03 smithi172 ceph-mon[25457]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:06.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:05 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:28:06.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:05 smithi019 ceph-mon[29267]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:06.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:05 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:28:06.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:05 smithi019 ceph-mon[24574]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:06.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:05 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:28:06.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:05 smithi172 ceph-mon[25457]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:07.992 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:07 smithi172 ceph-mon[25457]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:08.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:07 smithi019 ceph-mon[29267]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:08.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:07 smithi019 ceph-mon[24574]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:08.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:28:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:28:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:28:08.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:28:08 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:28:08] "GET /metrics HTTP/1.1" 200 207964 "" "Prometheus/2.33.4" 2024-02-07T12:28:09.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:08 smithi019 ceph-mon[24574]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:09.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:08 smithi019 ceph-mon[29267]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:08 smithi172 ceph-mon[25457]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:12.072 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:11 smithi172 ceph-mon[25457]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:12.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:11 smithi019 ceph-mon[24574]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:12.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:11 smithi019 ceph-mon[29267]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:14.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:13 smithi019 ceph-mon[29267]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:14.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:13 smithi019 ceph-mon[24574]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:14.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:13 smithi172 ceph-mon[25457]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:16.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:15 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:28:16.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:15 smithi019 ceph-mon[29267]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:16.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:15 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:28:16.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:15 smithi019 ceph-mon[24574]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:16.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:15 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:28:16.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:15 smithi172 ceph-mon[25457]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:17.992 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:17 smithi172 ceph-mon[25457]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:18.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:17 smithi019 ceph-mon[29267]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:18.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:17 smithi019 ceph-mon[24574]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:18.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:28:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:28:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:28:18.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:28:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:28:18] "GET /metrics HTTP/1.1" 200 207960 "" "Prometheus/2.33.4" 2024-02-07T12:28:19.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:18 smithi019 ceph-mon[24574]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:19.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:18 smithi019 ceph-mon[29267]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:19.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:18 smithi172 ceph-mon[25457]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:22.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:21 smithi019 ceph-mon[29267]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:22.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:21 smithi019 ceph-mon[24574]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:22.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:21 smithi172 ceph-mon[25457]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:24.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:23 smithi019 ceph-mon[29267]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:24.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:23 smithi019 ceph-mon[24574]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:24.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:23 smithi172 ceph-mon[25457]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:26.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:25 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:28:26.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:25 smithi019 ceph-mon[29267]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:26.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:25 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:28:26.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:25 smithi019 ceph-mon[24574]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:26.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:25 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:28:26.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:25 smithi172 ceph-mon[25457]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:27.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:27 smithi172 ceph-mon[25457]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:28.056 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:27 smithi019 ceph-mon[24574]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:28.056 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:27 smithi019 ceph-mon[29267]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:28.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:28:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:28:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:28:28.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:28:28 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:28:28] "GET /metrics HTTP/1.1" 200 207960 "" "Prometheus/2.33.4" 2024-02-07T12:28:29.259 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:28 smithi019 ceph-mon[29267]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:29.259 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:28 smithi019 ceph-mon[24574]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:29.262 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:28 smithi172 ceph-mon[25457]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:30.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:30 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:28:30.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:30 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:30.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:30 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:28:30.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:30 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:28:30.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:30 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:28:30.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:30 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:30.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:30 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:28:30.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:30 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:28:30.571 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:30 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:28:30.571 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:30 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:30.571 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:30 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:28:30.571 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:30 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:28:31.469 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:31 smithi019 ceph-mon[29267]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:31.469 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:31 smithi019 ceph-mon[24574]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:31.526 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:31 smithi172 ceph-mon[25457]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:33.753 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:33 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:28:33.753 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:33 smithi019 ceph-mon[29267]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:33.753 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:33 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:28:33.754 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:33 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:28:33.754 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:33 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:28:33.754 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:33 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:28:33.754 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:33 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:28:33.754 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:33 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:33.754 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:33 smithi019 ceph-mon[29267]: Deploying daemon mgr.x on smithi172 2024-02-07T12:28:33.754 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:33 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:28:33.754 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:33 smithi019 ceph-mon[24574]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:33.754 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:33 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:28:33.754 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:33 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:28:33.754 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:33 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:28:33.755 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:33 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:28:33.755 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:33 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:28:33.755 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:33 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:33.755 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:33 smithi019 ceph-mon[24574]: Deploying daemon mgr.x on smithi172 2024-02-07T12:28:33.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:33 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:28:33.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:33 smithi172 ceph-mon[25457]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:33.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:33 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:28:33.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:33 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:28:33.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:33 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:28:33.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:33 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:28:33.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:33 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:28:33.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:33 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:33.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:33 smithi172 ceph-mon[25457]: Deploying daemon mgr.x on smithi172 2024-02-07T12:28:36.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:35 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:28:36.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:35 smithi019 ceph-mon[24574]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:36.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:35 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:28:36.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:35 smithi019 ceph-mon[29267]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:36.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:35 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:28:36.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:35 smithi172 ceph-mon[25457]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:37.994 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:37 smithi172 ceph-mon[25457]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:38.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:37 smithi019 ceph-mon[29267]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:38.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:37 smithi019 ceph-mon[24574]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:38.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:28:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:28:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:28:38.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:28:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:28:38] "GET /metrics HTTP/1.1" 200 207964 "" "Prometheus/2.33.4" 2024-02-07T12:28:39.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:38 smithi019 ceph-mon[24574]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:39.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:38 smithi019 ceph-mon[29267]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:39.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:38 smithi172 ceph-mon[25457]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:42.071 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:41 smithi172 ceph-mon[25457]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:42.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:41 smithi019 ceph-mon[24574]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:42.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:41 smithi019 ceph-mon[29267]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:43.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:43 smithi019 ceph-mon[29267]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:43.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:43 smithi019 ceph-mon[24574]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:43.845 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:28:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:28:43.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs" 2024-02-07T12:28:43.845 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:28:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:28:43.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:28:43.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:43 smithi172 ceph-mon[25457]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:46.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:45 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:28:46.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:45 smithi019 ceph-mon[29267]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:46.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:45 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:28:46.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:45 smithi019 ceph-mon[24574]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:46.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:45 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:28:46.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:45 smithi172 ceph-mon[25457]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:47.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:46 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:28:47.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:46 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:28:47.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:46 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:28:47.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:46 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:28:47.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:46 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:28:47.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:46 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:28:47.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:46 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:28:47.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:46 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:28:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:46 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:28:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:46 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:28:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:46 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:28:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:46 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:28:47.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:47 smithi172 ceph-mon[25457]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:48.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:47 smithi019 ceph-mon[29267]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:48.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:47 smithi019 ceph-mon[24574]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:48.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:28:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:28:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:28:48.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:28:48 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:28:48] "GET /metrics HTTP/1.1" 200 207969 "" "Prometheus/2.33.4" 2024-02-07T12:28:49.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:48 smithi019 ceph-mon[24574]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:49.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:48 smithi019 ceph-mon[29267]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:49.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:48 smithi172 ceph-mon[25457]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:52.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:51 smithi019 ceph-mon[29267]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:52.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:51 smithi019 ceph-mon[24574]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:52.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:51 smithi172 ceph-mon[25457]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:53.772 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:53 smithi019 ceph-mon[29267]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:53.772 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:53 smithi019 ceph-mon[24574]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:53.773 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:28:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=error ts=2024-02-07T12:28:53.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs" 2024-02-07T12:28:53.773 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:28:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:28:53.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:28:53.773 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:28:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:28:53.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs" 2024-02-07T12:28:53.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:53 smithi172 ceph-mon[25457]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:56.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:55 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:28:56.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:55 smithi019 ceph-mon[24574]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:56.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:55 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:28:56.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:55 smithi019 ceph-mon[29267]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:56.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:55 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:28:56.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:55 smithi172 ceph-mon[25457]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:57.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:57 smithi172 ceph-mon[25457]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:58.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:57 smithi019 ceph-mon[29267]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:58.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:57 smithi019 ceph-mon[24574]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:28:58.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:28:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[26640]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:28:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:28:58.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:28:58 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:28:58] "GET /metrics HTTP/1.1" 200 207969 "" "Prometheus/2.33.4" 2024-02-07T12:28:59.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:28:58 smithi019 ceph-mon[24574]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:59.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:28:58 smithi019 ceph-mon[29267]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:28:59.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:28:58 smithi172 ceph-mon[25457]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:01.754 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:01 smithi172 ceph-mon[25457]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:02.048 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:01 smithi019 ceph-mon[29267]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:02.048 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:01 smithi019 ceph-mon[24574]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:02.805 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:02 smithi172 systemd[1]: Stopping Ceph mgr.x for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:29:03.119 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:02 smithi172 bash[45490]: Error: no container with name or ID "ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr.x" found: no such container 2024-02-07T12:29:03.119 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:02 smithi172 podman[45503]: 2024-02-07 12:29:02.939244359 +0000 UTC m=+0.127242190 container died 3ce4fcd95143d8b5bc16a2f80bf7ef24e04f500cb7768ceedec624afd4898c24 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x, maintainer=Guillaume Abrioux , name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, vendor=Red Hat, Inc., vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, io.openshift.expose-services=, GIT_CLEAN=True, GIT_BRANCH=HEAD, version=8, ceph=True) 2024-02-07T12:29:03.555 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:03 smithi172 podman[45503]: 2024-02-07 12:29:03.360400302 +0000 UTC m=+0.548398182 container cleanup 3ce4fcd95143d8b5bc16a2f80bf7ef24e04f500cb7768ceedec624afd4898c24 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, name=centos-stream, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, RELEASE=HEAD, io.buildah.version=1.19.8, release=754, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-02-07T12:29:03.555 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:03 smithi172 bash[45503]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x 2024-02-07T12:29:03.555 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:03 smithi172 bash[45529]: Error: no container with name or ID "ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr.x" found: no such container 2024-02-07T12:29:03.781 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:03 smithi019 ceph-mon[24574]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:03.782 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:03 smithi019 ceph-mon[29267]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:03.782 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:29:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=error ts=2024-02-07T12:29:03.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:29:03.782 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:29:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:29:03.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": dial tcp 172.21.15.172:8443: connect: connection refused" 2024-02-07T12:29:03.782 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:29:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:29:03.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs" 2024-02-07T12:29:03.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:03 smithi172 ceph-mon[25457]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:03.870 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:03 smithi172 podman[45517]: 2024-02-07 12:29:03.553915152 +0000 UTC m=+0.611339606 container remove 3ce4fcd95143d8b5bc16a2f80bf7ef24e04f500cb7768ceedec624afd4898c24 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.openshift.expose-services=, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, io.openshift.tags=base centos centos-stream, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, name=centos-stream, maintainer=Guillaume Abrioux , RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-02-07T12:29:03.870 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:03 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-02-07T12:29:03.870 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:03 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mgr.x.service: Failed with result 'exit-code'. 2024-02-07T12:29:03.870 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:03 smithi172 systemd[1]: Stopped Ceph mgr.x for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:29:03.870 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:03 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mgr.x.service: Consumed 17.051s CPU time. 2024-02-07T12:29:03.870 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:03 smithi172 systemd[1]: Starting Ceph mgr.x for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:29:04.284 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:03 smithi172 podman[45629]: 2024-02-07 12:29:03.870684103 +0000 UTC m=+0.019547049 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:29:04.284 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:03 smithi172 podman[45629]: 2024-02-07 12:29:03.971634182 +0000 UTC m=+0.120497118 container create 97644267e529cd5b3f535ef8aef0c569f9aa23b49b12169ac15caabec0172c60 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:29:04.284 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:04 smithi172 podman[45629]: 2024-02-07 12:29:04.182291522 +0000 UTC m=+0.331154458 container init 97644267e529cd5b3f535ef8aef0c569f9aa23b49b12169ac15caabec0172c60 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, io.buildah.version=1.33.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-02-07T12:29:04.284 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:04 smithi172 podman[45629]: 2024-02-07 12:29:04.184920348 +0000 UTC m=+0.333783285 container start 97644267e529cd5b3f535ef8aef0c569f9aa23b49b12169ac15caabec0172c60 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-02-07T12:29:04.284 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:04 smithi172 ceph-mgr[45648]: -- 172.21.15.172:0/1903838949 <== mon.2 v2:172.21.15.172:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55dc72a4b520 con 0x55dc72a20800 2024-02-07T12:29:04.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:04 smithi172 bash[45629]: 97644267e529cd5b3f535ef8aef0c569f9aa23b49b12169ac15caabec0172c60 2024-02-07T12:29:04.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:04 smithi172 systemd[1]: Started Ceph mgr.x for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:29:05.119 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:29:04.756+0000 7f50ab9e1200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-02-07T12:29:05.119 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:29:04.854+0000 7f50ab9e1200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-02-07T12:29:05.541 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:05 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:29:05.541 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:05 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:29:05.541 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:05 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:29:05.541 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:05 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:29:05.542 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:05 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:29:05.542 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:05 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:29:05.542 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:05 smithi172 ceph-mon[25457]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:05.542 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:05 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:29:05.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:05 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:29:05.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:05 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:29:05.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:05 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:29:05.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:05 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:29:05.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:05 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:29:05.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:05 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:29:05.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:05 smithi019 ceph-mon[24574]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:05.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:05 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:29:05.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:05 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:29:05.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:05 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:29:05.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:05 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:29:05.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:05 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:29:05.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:05 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:29:05.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:05 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:29:05.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:05 smithi019 ceph-mon[29267]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:05.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:05 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:29:05.869 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:05 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:29:05.538+0000 7f50ab9e1200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-02-07T12:29:05.869 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:05 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:29:05.631+0000 7f50ab9e1200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-02-07T12:29:05.869 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:05 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:29:05.787+0000 7f50ab9e1200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-02-07T12:29:06.148 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:05 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:29:05.888+0000 7f50ab9e1200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-02-07T12:29:06.148 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:05 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:29:05.977+0000 7f50ab9e1200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-02-07T12:29:06.481 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:06 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:29:06.260+0000 7f50ab9e1200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-02-07T12:29:06.481 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:06 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:29:06.386+0000 7f50ab9e1200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-02-07T12:29:06.764 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:06 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:29:06.477+0000 7f50ab9e1200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-02-07T12:29:06.764 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:06 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:29:06.576+0000 7f50ab9e1200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-02-07T12:29:07.119 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:06 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:29:06.761+0000 7f50ab9e1200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-02-07T12:29:07.912 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:07 smithi172 ceph-mon[25457]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:07.913 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:29:07.778+0000 7f50ab9e1200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-02-07T12:29:08.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:07 smithi019 ceph-mon[29267]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:08.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:07 smithi019 ceph-mon[24574]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:08.542 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:29:07.909+0000 7f50ab9e1200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-02-07T12:29:08.542 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:08 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:29:08.250+0000 7f50ab9e1200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-02-07T12:29:08.542 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:08 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:29:08.342+0000 7f50ab9e1200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-02-07T12:29:08.542 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:08 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:29:08.539+0000 7f50ab9e1200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:29:08.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:29:08 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:29:08] "GET /metrics HTTP/1.1" 200 207977 "" "Prometheus/2.33.4" 2024-02-07T12:29:08.869 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:08 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:29:08.725+0000 7f50ab9e1200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-02-07T12:29:09.290 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:09 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:29:09.012+0000 7f50ab9e1200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:29:09.290 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:08 smithi172 ceph-mon[25457]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:09.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:08 smithi019 ceph-mon[24574]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:09.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:08 smithi019 ceph-mon[29267]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:09.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:09 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:29:09.277+0000 7f50ab9e1200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-02-07T12:29:09.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:09 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:29:09.525+0000 7f50ab9e1200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-02-07T12:29:10.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:10 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:29:10.339+0000 7f50ab9e1200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-02-07T12:29:11.242 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:10 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:29:10.921+0000 7f50ab9e1200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-02-07T12:29:11.242 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:11 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:29:11.021+0000 7f50ab9e1200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-02-07T12:29:11.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:11 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:29:11.240+0000 7f50ab9e1200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-02-07T12:29:11.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:11 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:29:11] ENGINE Bus STARTING 2024-02-07T12:29:11.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:11 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: CherryPy Checker: 2024-02-07T12:29:11.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:11 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: The Application mounted at '' has an empty config. 2024-02-07T12:29:11.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:11 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:29:11.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:11 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:29:11] ENGINE Serving on http://:::9283 2024-02-07T12:29:11.620 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:11 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:29:11] ENGINE Bus STARTED 2024-02-07T12:29:12.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:11 smithi019 ceph-mon[24574]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:12.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:11 smithi019 ceph-mon[24574]: Standby manager daemon x restarted 2024-02-07T12:29:12.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:11 smithi019 ceph-mon[24574]: Standby manager daemon x started 2024-02-07T12:29:12.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:11 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-02-07T12:29:12.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:11 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:29:12.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:11 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-02-07T12:29:12.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:11 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:29:12.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:11 smithi019 ceph-mon[29267]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:12.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:11 smithi019 ceph-mon[29267]: Standby manager daemon x restarted 2024-02-07T12:29:12.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:11 smithi019 ceph-mon[29267]: Standby manager daemon x started 2024-02-07T12:29:12.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:11 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-02-07T12:29:12.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:11 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:29:12.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:11 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-02-07T12:29:12.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:11 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:29:12.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:11 smithi172 ceph-mon[25457]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:12.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:11 smithi172 ceph-mon[25457]: Standby manager daemon x restarted 2024-02-07T12:29:12.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:11 smithi172 ceph-mon[25457]: Standby manager daemon x started 2024-02-07T12:29:12.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:11 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-02-07T12:29:12.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:11 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:29:12.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:11 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-02-07T12:29:12.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:11 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:29:13.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:12 smithi019 ceph-mon[24574]: mgrmap e22: y(active, since 6m), standbys: x 2024-02-07T12:29:13.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:12 smithi019 ceph-mon[29267]: mgrmap e22: y(active, since 6m), standbys: x 2024-02-07T12:29:13.095 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:29:12 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:29:12.859Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=7 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:29:13.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:12 smithi172 ceph-mon[25457]: mgrmap e22: y(active, since 6m), standbys: x 2024-02-07T12:29:13.785 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:13 smithi019 ceph-mon[29267]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:13.786 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:13 smithi019 ceph-mon[24574]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:13.786 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:29:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=error ts=2024-02-07T12:29:13.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:29:13.786 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:29:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:29:13.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs" 2024-02-07T12:29:13.786 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:29:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:29:13.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:29:14.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:13 smithi172 ceph-mon[25457]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:16.025 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:15 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:29:16.025 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:15 smithi172 ceph-mon[25457]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:16.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:15 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:29:16.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:15 smithi019 ceph-mon[29267]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:16.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:15 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:29:16.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:15 smithi019 ceph-mon[24574]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:17.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:17 smithi172 ceph-mon[25457]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:18.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:17 smithi019 ceph-mon[29267]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:18.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:17 smithi019 ceph-mon[24574]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:18.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:29:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:29:18.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:29:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:29:18] "GET /metrics HTTP/1.1" 200 207980 "" "Prometheus/2.33.4" 2024-02-07T12:29:19.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:18 smithi019 ceph-mon[24574]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:19.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:18 smithi019 ceph-mon[29267]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:19.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:18 smithi172 ceph-mon[25457]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:21.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:20 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:29:21.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:20 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:29:21.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:20 smithi019 ceph-mon[24574]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:21.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:20 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:29:21.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:20 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:29:21.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:20 smithi019 ceph-mon[29267]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:20 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:29:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:20 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:29:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:20 smithi172 ceph-mon[25457]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:23.791 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:23 smithi019 ceph-mon[24574]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:23.792 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:23 smithi019 ceph-mon[29267]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:23.792 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:29:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=error ts=2024-02-07T12:29:23.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:29:23.792 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:29:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:29:23.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs" 2024-02-07T12:29:23.792 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:29:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:29:23.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:29:24.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:23 smithi172 ceph-mon[25457]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:26.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:25 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:29:26.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:25 smithi019 ceph-mon[24574]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:26.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:25 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:29:26.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:25 smithi019 ceph-mon[29267]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:26.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:25 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:29:26.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:25 smithi172 ceph-mon[25457]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:27.994 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:27 smithi172 ceph-mon[25457]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:28.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:27 smithi019 ceph-mon[24574]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:28.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:27 smithi019 ceph-mon[29267]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:28.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:29:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:29:28.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:29:28 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:29:28] "GET /metrics HTTP/1.1" 200 207980 "" "Prometheus/2.33.4" 2024-02-07T12:29:29.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:28 smithi019 ceph-mon[24574]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:29.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:28 smithi019 ceph-mon[29267]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:28 smithi172 ceph-mon[25457]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:32.048 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:31 smithi019 ceph-mon[29267]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:32.048 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:31 smithi019 ceph-mon[24574]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:32.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:31 smithi172 ceph-mon[25457]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:33.803 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:33 smithi019 ceph-mon[29267]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:33.804 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:33 smithi019 ceph-mon[24574]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:33.804 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:29:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=error ts=2024-02-07T12:29:33.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs" 2024-02-07T12:29:33.804 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:29:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:29:33.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs" 2024-02-07T12:29:33.804 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:29:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:29:33.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:29:34.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:33 smithi172 ceph-mon[25457]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:36.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:35 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:29:36.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:35 smithi019 ceph-mon[29267]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:36.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:35 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:29:36.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:35 smithi019 ceph-mon[24574]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:36.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:35 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:29:36.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:35 smithi172 ceph-mon[25457]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:37.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:37 smithi172 ceph-mon[25457]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:38.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:37 smithi019 ceph-mon[29267]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:38.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:37 smithi019 ceph-mon[24574]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:38.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:29:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:29:38.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:29:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:29:38] "GET /metrics HTTP/1.1" 200 207984 "" "Prometheus/2.33.4" 2024-02-07T12:29:39.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:38 smithi019 ceph-mon[29267]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:39.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:38 smithi019 ceph-mon[24574]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:39.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:38 smithi172 ceph-mon[25457]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:42.072 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:41 smithi172 ceph-mon[25457]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:42.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:41 smithi019 ceph-mon[29267]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:42.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:41 smithi019 ceph-mon[24574]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:43.812 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:43 smithi019 ceph-mon[29267]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:43.812 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:43 smithi019 ceph-mon[24574]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:43.813 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:29:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=error ts=2024-02-07T12:29:43.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:29:43.813 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:29:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:29:43.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:29:43.813 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:29:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:29:43.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs" 2024-02-07T12:29:43.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:43 smithi172 ceph-mon[25457]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:46.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:45 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:29:46.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:45 smithi019 ceph-mon[29267]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:46.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:45 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:29:46.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:45 smithi019 ceph-mon[24574]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:46.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:45 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:29:46.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:45 smithi172 ceph-mon[25457]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:47.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:46 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:29:47.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:46 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:29:47.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:46 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:29:47.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:46 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:29:47.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:46 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:29:47.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:46 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:29:47.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:46 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:29:47.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:46 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:29:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:46 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:29:47.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:46 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:29:47.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:46 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:29:47.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:46 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:29:47.994 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:47 smithi172 ceph-mon[25457]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:48.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:47 smithi019 ceph-mon[29267]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:48.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:47 smithi019 ceph-mon[24574]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:48.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:29:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:29:48.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:29:48 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:29:48] "GET /metrics HTTP/1.1" 200 207983 "" "Prometheus/2.33.4" 2024-02-07T12:29:49.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:49 smithi019 ceph-mon[29267]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:49.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:49 smithi019 ceph-mon[24574]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:49.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:49 smithi172 ceph-mon[25457]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:52.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:51 smithi019 ceph-mon[24574]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:52.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:51 smithi019 ceph-mon[29267]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:52.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:51 smithi172 ceph-mon[25457]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:53.820 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:53 smithi019 ceph-mon[24574]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:53.820 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:53 smithi019 ceph-mon[29267]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:53.821 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:29:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=error ts=2024-02-07T12:29:53.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:29:53.821 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:29:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:29:53.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs" 2024-02-07T12:29:53.821 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:29:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:29:53.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:29:54.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:53 smithi172 ceph-mon[25457]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:56.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:55 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:29:56.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:55 smithi019 ceph-mon[24574]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:56.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:55 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:29:56.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:55 smithi019 ceph-mon[29267]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:56.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:55 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:29:56.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:55 smithi172 ceph-mon[25457]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:57.992 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:57 smithi172 ceph-mon[25457]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:58.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:57 smithi019 ceph-mon[24574]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:58.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:57 smithi019 ceph-mon[29267]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:29:58.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:29:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:29:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:29:58.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:29:58 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:29:58] "GET /metrics HTTP/1.1" 200 207983 "" "Prometheus/2.33.4" 2024-02-07T12:29:59.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:29:59 smithi019 ceph-mon[24574]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:59.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:29:59 smithi019 ceph-mon[29267]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:29:59.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:29:59 smithi172 ceph-mon[25457]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:00.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:00 smithi019 ceph-mon[24574]: overall HEALTH_OK 2024-02-07T12:30:00.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:00 smithi019 ceph-mon[29267]: overall HEALTH_OK 2024-02-07T12:30:00.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:00 smithi172 ceph-mon[25457]: overall HEALTH_OK 2024-02-07T12:30:01.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:01 smithi019 ceph-mon[29267]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:01.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:01 smithi019 ceph-mon[24574]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:01.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:01 smithi172 ceph-mon[25457]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:03.830 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:03 smithi019 ceph-mon[24574]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:03.831 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:03 smithi019 ceph-mon[29267]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:03.831 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:30:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=error ts=2024-02-07T12:30:03.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs" 2024-02-07T12:30:03.831 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:30:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:30:03.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs" 2024-02-07T12:30:03.831 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:30:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:30:03.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:30:04.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:03 smithi172 ceph-mon[25457]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:06.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:05 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:30:06.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:05 smithi019 ceph-mon[24574]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:06.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:05 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:30:06.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:05 smithi019 ceph-mon[29267]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:06.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:05 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:30:06.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:05 smithi172 ceph-mon[25457]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:07.992 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:07 smithi172 ceph-mon[25457]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:08.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:07 smithi019 ceph-mon[24574]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:08.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:07 smithi019 ceph-mon[29267]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:08.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:30:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:30:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:30:08.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:30:08 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:30:08] "GET /metrics HTTP/1.1" 200 207976 "" "Prometheus/2.33.4" 2024-02-07T12:30:09.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:09 smithi019 ceph-mon[24574]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:09.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:09 smithi019 ceph-mon[29267]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:09 smithi172 ceph-mon[25457]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:12.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:11 smithi019 ceph-mon[29267]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:12.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:11 smithi019 ceph-mon[24574]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:12.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:11 smithi172 ceph-mon[25457]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:13.841 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:13 smithi019 ceph-mon[29267]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:13.841 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:13 smithi019 ceph-mon[24574]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:13.842 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:30:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=error ts=2024-02-07T12:30:13.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs" 2024-02-07T12:30:13.842 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:30:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:30:13.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs" 2024-02-07T12:30:13.842 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:30:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:30:13.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:30:14.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:13 smithi172 ceph-mon[25457]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:16.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:15 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:30:16.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:15 smithi019 ceph-mon[24574]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:16.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:15 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:30:16.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:15 smithi019 ceph-mon[29267]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:16.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:15 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:30:16.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:15 smithi172 ceph-mon[25457]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:17.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:17 smithi172 ceph-mon[25457]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:18.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:17 smithi019 ceph-mon[29267]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:18.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:17 smithi019 ceph-mon[24574]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:18.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:30:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:30:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:30:18.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:30:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:30:18] "GET /metrics HTTP/1.1" 200 207984 "" "Prometheus/2.33.4" 2024-02-07T12:30:19.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:19 smithi019 ceph-mon[29267]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:19.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:19 smithi019 ceph-mon[24574]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:19.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:19 smithi172 ceph-mon[25457]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:20.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:20 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:30:20.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:20 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:30:20.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:20 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:30:20.656 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:20 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:30:20.656 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:20 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:30:20.656 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:20 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:30:20.656 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:20 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:30:20.656 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:20 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:30:20.657 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:20 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:30:22.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:21 smithi019 ceph-mon[29267]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:22.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:21 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:30:22.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:21 smithi019 ceph-mon[24574]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:22.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:21 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:30:22.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:21 smithi172 ceph-mon[25457]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:22.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:21 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' 2024-02-07T12:30:23.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:23 smithi019 ceph-mon[29267]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:23.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:23 smithi019 ceph-mon[24574]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:23.845 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:30:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=error ts=2024-02-07T12:30:23.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:30:23.845 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:30:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:30:23.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs" 2024-02-07T12:30:23.845 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:30:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:30:23.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:30:24.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:23 smithi172 ceph-mon[25457]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:26.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:25 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:30:26.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:25 smithi019 ceph-mon[24574]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:26.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:25 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:30:26.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:25 smithi019 ceph-mon[29267]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:26.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:25 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:30:26.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:25 smithi172 ceph-mon[25457]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:27.992 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:27 smithi172 ceph-mon[25457]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:28.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:27 smithi019 ceph-mon[24574]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:28.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:27 smithi019 ceph-mon[29267]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:28.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:30:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:30:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:30:28.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:30:28 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:30:28] "GET /metrics HTTP/1.1" 200 207984 "" "Prometheus/2.33.4" 2024-02-07T12:30:29.347 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:29 smithi019 ceph-mon[24574]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:29.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:29 smithi019 ceph-mon[29267]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:29 smithi172 ceph-mon[25457]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:32.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:31 smithi019 ceph-mon[24574]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:32.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:31 smithi019 ceph-mon[29267]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:32.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:31 smithi172 ceph-mon[25457]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:33.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:33 smithi019 ceph-mon[24574]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:33.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:33 smithi019 ceph-mon[29267]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:33.845 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:30:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=error ts=2024-02-07T12:30:33.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:30:33.845 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:30:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:30:33.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs" 2024-02-07T12:30:33.846 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:30:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:30:33.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:30:34.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:33 smithi172 ceph-mon[25457]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:36.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:35 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:30:36.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:35 smithi019 ceph-mon[29267]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:36.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:35 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:30:36.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:35 smithi019 ceph-mon[24574]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:36.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:35 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:30:36.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:35 smithi172 ceph-mon[25457]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:37.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:37 smithi172 ceph-mon[25457]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:38.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:37 smithi019 ceph-mon[29267]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:38.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:37 smithi019 ceph-mon[24574]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:38.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:30:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:30:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:30:38.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:30:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:30:38] "GET /metrics HTTP/1.1" 200 207978 "" "Prometheus/2.33.4" 2024-02-07T12:30:39.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:39 smithi019 ceph-mon[24574]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:39.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:39 smithi019 ceph-mon[29267]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:39.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:39 smithi172 ceph-mon[25457]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:42.076 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:41 smithi172 ceph-mon[25457]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:42.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:41 smithi019 ceph-mon[24574]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:42.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:41 smithi019 ceph-mon[29267]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:43.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:43 smithi019 ceph-mon[24574]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:43.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:43 smithi019 ceph-mon[29267]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:43.845 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:30:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=error ts=2024-02-07T12:30:43.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:30:43.845 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:30:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:30:43.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs" 2024-02-07T12:30:43.845 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:30:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:30:43.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:30:44.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:43 smithi172 ceph-mon[25457]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:46.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:45 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:30:46.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:45 smithi019 ceph-mon[24574]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:46.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:45 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:30:46.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:45 smithi019 ceph-mon[29267]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:46.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:45 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:30:46.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:45 smithi172 ceph-mon[25457]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:47.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:46 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:30:47.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:46 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:30:47.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:46 smithi019 ceph-mon[24574]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:30:47.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:46 smithi019 ceph-mon[24574]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:30:47.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:46 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:30:47.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:46 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:30:47.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:46 smithi019 ceph-mon[29267]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:30:47.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:46 smithi019 ceph-mon[29267]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:30:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:46 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:30:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:46 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:30:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:46 smithi172 ceph-mon[25457]: from='mgr.14433 172.21.15.19:0/1730045700' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:30:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:46 smithi172 ceph-mon[25457]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:30:47.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:47 smithi172 ceph-mon[25457]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:48.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:47 smithi019 ceph-mon[24574]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:48.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:47 smithi019 ceph-mon[29267]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:48.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:30:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:30:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:30:48.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:30:48 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:30:48] "GET /metrics HTTP/1.1" 200 207986 "" "Prometheus/2.33.4" 2024-02-07T12:30:49.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:49 smithi019 ceph-mon[29267]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:49.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:49 smithi019 ceph-mon[24574]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:49.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:49 smithi172 ceph-mon[25457]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:52.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:51 smithi019 ceph-mon[29267]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:52.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:51 smithi019 ceph-mon[24574]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:52.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:51 smithi172 ceph-mon[25457]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:53.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:53 smithi019 ceph-mon[29267]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:53.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:53 smithi019 ceph-mon[24574]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:53.845 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:30:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=error ts=2024-02-07T12:30:53.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:30:53.845 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:30:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:30:53.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs" 2024-02-07T12:30:53.845 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:30:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:30:53.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:30:54.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:53 smithi172 ceph-mon[25457]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:56.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:55 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:30:56.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:55 smithi019 ceph-mon[24574]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:56.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:55 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:30:56.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:55 smithi019 ceph-mon[29267]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:56.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:55 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:30:56.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:55 smithi172 ceph-mon[25457]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:57.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:57 smithi172 ceph-mon[25457]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:58.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:57 smithi019 ceph-mon[29267]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:58.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:57 smithi019 ceph-mon[24574]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:30:58.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:30:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:30:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:30:58.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:30:58 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:30:58] "GET /metrics HTTP/1.1" 200 207986 "" "Prometheus/2.33.4" 2024-02-07T12:30:59.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:30:59 smithi019 ceph-mon[24574]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:59.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:30:59 smithi019 ceph-mon[29267]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:30:59.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:30:59 smithi172 ceph-mon[25457]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:31:02.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:01 smithi019 ceph-mon[24574]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:31:02.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:01 smithi019 ceph-mon[29267]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:31:02.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:01 smithi172 ceph-mon[25457]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:31:03.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:03 smithi019 ceph-mon[24574]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:31:03.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:03 smithi019 ceph-mon[29267]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:31:03.845 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=error ts=2024-02-07T12:31:03.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs" 2024-02-07T12:31:03.845 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:31:03.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs" 2024-02-07T12:31:03.845 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:31:03.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:31:04.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:03 smithi172 ceph-mon[25457]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:31:05.203 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch ps' 2024-02-07T12:31:05.818 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:05 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:31:05.818 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:05 smithi019 ceph-mon[24574]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:31:05.818 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:05 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:31:05.818 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:05 smithi019 ceph-mon[29267]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:31:06.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:05 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:31:06.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:05 smithi172 ceph-mon[25457]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:31:07.302 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:31:07.302 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (6m) 2m ago 7m 23.4M - ba2b418f427c 4eb1830b3c20 2024-02-07T12:31:07.302 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (7m) 107s ago 6m 55.6M - 8.3.5 dad864ee21e9 fcafad07f161 2024-02-07T12:31:07.302 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 running (6m) 2m ago 6m 53.3M - 3.5 e1d6a67b021e 32cb504d4898 2024-02-07T12:31:07.302 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283 running (2m) 107s ago 11m 68.5M - 19.0.0-1252-g44458cce 79957b372307 97644267e529 2024-02-07T12:31:07.302 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:9283 running (13m) 2m ago 13m 460M - 17.2.0 e1d6a67b021e c025f688a668 2024-02-07T12:31:07.302 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (13m) 2m ago 13m 51.9M 2048M 17.2.0 e1d6a67b021e f3a8a050cf1d 2024-02-07T12:31:07.302 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (11m) 107s ago 11m 37.5M 2048M 17.2.0 e1d6a67b021e 7d62c7c4d075 2024-02-07T12:31:07.302 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (12m) 2m ago 12m 39.8M 2048M 17.2.0 e1d6a67b021e b9825d0db32d 2024-02-07T12:31:07.303 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (7m) 2m ago 7m 16.8M - 1dbe0e931976 de9599dbfeff 2024-02-07T12:31:07.303 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (7m) 107s ago 7m 17.1M - 1dbe0e931976 30f1dfd23ff2 2024-02-07T12:31:07.303 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (11m) 2m ago 11m 49.9M 4017M 17.2.0 e1d6a67b021e 87f2560a9b15 2024-02-07T12:31:07.303 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (11m) 2m ago 11m 52.3M 4017M 17.2.0 e1d6a67b021e 0f9a190f0e0d 2024-02-07T12:31:07.303 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (10m) 2m ago 10m 48.4M 4017M 17.2.0 e1d6a67b021e 7db94f25a1aa 2024-02-07T12:31:07.303 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (10m) 2m ago 10m 48.9M 4017M 17.2.0 e1d6a67b021e 0b6b674188ed 2024-02-07T12:31:07.303 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (9m) 107s ago 9m 49.0M 4272M 17.2.0 e1d6a67b021e df72cfd69b3b 2024-02-07T12:31:07.303 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (9m) 107s ago 9m 47.7M 4272M 17.2.0 e1d6a67b021e 3bdb22c07537 2024-02-07T12:31:07.303 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (9m) 107s ago 9m 46.0M 4272M 17.2.0 e1d6a67b021e 7859b651e0d9 2024-02-07T12:31:07.303 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (8m) 107s ago 8m 50.6M 4272M 17.2.0 e1d6a67b021e b301e7c75d73 2024-02-07T12:31:07.303 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (6m) 107s ago 8m 66.3M - 514e6a882f6e d482181f6c26 2024-02-07T12:31:07.303 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (6m) 2m ago 6m 72.3M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:31:07.303 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (6m) 107s ago 6m 71.7M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:31:07.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:07 smithi019 ceph-mon[24574]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:31:07.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:07 smithi019 ceph-mon[29267]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:31:07.919 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph versions' 2024-02-07T12:31:07.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:07 smithi172 ceph-mon[25457]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:31:08.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:31:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:31:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:31:08.523 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:08 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:31:08] "GET /metrics HTTP/1.1" 200 207988 "" "Prometheus/2.33.4" 2024-02-07T12:31:08.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:08 smithi019 ceph-mon[24574]: from='client.24904 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:31:08.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:08 smithi019 ceph-mon[29267]: from='client.24904 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:31:09.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:08 smithi172 ceph-mon[25457]: from='client.24904 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:31:09.830 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:09 smithi019 ceph-mon[29267]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:31:09.831 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:09 smithi019 ceph-mon[24574]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:31:10.114 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:31:10.114 INFO:teuthology.orchestra.run.smithi019.stdout: "mon": { 2024-02-07T12:31:10.114 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-02-07T12:31:10.114 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:31:10.114 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr": { 2024-02-07T12:31:10.114 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-02-07T12:31:10.115 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 1 2024-02-07T12:31:10.115 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:31:10.115 INFO:teuthology.orchestra.run.smithi019.stdout: "osd": { 2024-02-07T12:31:10.115 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-02-07T12:31:10.115 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:31:10.115 INFO:teuthology.orchestra.run.smithi019.stdout: "mds": {}, 2024-02-07T12:31:10.115 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw": { 2024-02-07T12:31:10.115 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:31:10.115 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:31:10.115 INFO:teuthology.orchestra.run.smithi019.stdout: "overall": { 2024-02-07T12:31:10.115 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14, 2024-02-07T12:31:10.115 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 1 2024-02-07T12:31:10.115 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:31:10.115 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:31:10.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:09 smithi172 ceph-mon[25457]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:31:10.675 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph -s' 2024-02-07T12:31:10.987 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:10 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1852885441' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:31:10.988 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:10 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1852885441' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:31:11.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:10 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1852885441' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:31:11.916 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:11 smithi019 ceph-mon[29267]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:31:11.916 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:11 smithi019 ceph-mon[24574]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:31:12.020 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:11 smithi172 ceph-mon[25457]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:31:12.737 INFO:teuthology.orchestra.run.smithi019.stdout: cluster: 2024-02-07T12:31:12.737 INFO:teuthology.orchestra.run.smithi019.stdout: id: ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:31:12.737 INFO:teuthology.orchestra.run.smithi019.stdout: health: HEALTH_OK 2024-02-07T12:31:12.737 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:31:12.737 INFO:teuthology.orchestra.run.smithi019.stdout: services: 2024-02-07T12:31:12.737 INFO:teuthology.orchestra.run.smithi019.stdout: mon: 3 daemons, quorum a,c,b (age 11m) 2024-02-07T12:31:12.737 INFO:teuthology.orchestra.run.smithi019.stdout: mgr: y(active, since 8m), standbys: x 2024-02-07T12:31:12.737 INFO:teuthology.orchestra.run.smithi019.stdout: osd: 8 osds: 8 up (since 8m), 8 in (since 8m) 2024-02-07T12:31:12.737 INFO:teuthology.orchestra.run.smithi019.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-02-07T12:31:12.737 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:31:12.737 INFO:teuthology.orchestra.run.smithi019.stdout: data: 2024-02-07T12:31:12.737 INFO:teuthology.orchestra.run.smithi019.stdout: pools: 6 pools, 161 pgs 2024-02-07T12:31:12.738 INFO:teuthology.orchestra.run.smithi019.stdout: objects: 209 objects, 585 KiB 2024-02-07T12:31:12.738 INFO:teuthology.orchestra.run.smithi019.stdout: usage: 63 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:31:12.738 INFO:teuthology.orchestra.run.smithi019.stdout: pgs: 161 active+clean 2024-02-07T12:31:12.738 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:31:12.738 INFO:teuthology.orchestra.run.smithi019.stdout: io: 2024-02-07T12:31:12.738 INFO:teuthology.orchestra.run.smithi019.stdout: client: 1.2 KiB/s rd, 1 op/s rd, 0 op/s wr 2024-02-07T12:31:12.738 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:31:13.365 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph health detail' 2024-02-07T12:31:13.594 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=error ts=2024-02-07T12:31:13.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs" 2024-02-07T12:31:13.594 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:31:13.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:31:13.594 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:31:13.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs" 2024-02-07T12:31:13.960 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:13 smithi019 ceph-mon[24574]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:31:13.961 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:13 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1898486826' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-02-07T12:31:13.961 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:13 smithi019 ceph-mon[29267]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:31:13.961 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:13 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1898486826' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-02-07T12:31:14.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:13 smithi172 ceph-mon[25457]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:31:14.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:13 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1898486826' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-02-07T12:31:15.189 INFO:teuthology.orchestra.run.smithi019.stdout:HEALTH_OK 2024-02-07T12:31:15.755 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 2'"'"'' 2024-02-07T12:31:16.063 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:15 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:31:16.063 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:15 smithi019 ceph-mon[29267]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:31:16.064 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:15 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3663068261' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:31:16.064 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:15 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:31:16.064 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:15 smithi019 ceph-mon[24574]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:31:16.064 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:15 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3663068261' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:31:16.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:15 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:31:16.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:15 smithi172 ceph-mon[25457]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:31:16.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:15 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3663068261' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:31:17.793 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:31:17.980 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:17 smithi019 ceph-mon[24574]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:31:17.981 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:17 smithi019 ceph-mon[29267]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:31:17.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:17 smithi172 ceph-mon[25457]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:31:18.352 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph mgr fail' 2024-02-07T12:31:18.371 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:31:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:31:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:31:18.591 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:31:18] "GET /metrics HTTP/1.1" 200 207984 "" "Prometheus/2.33.4" 2024-02-07T12:31:18.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:18 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1972144434' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:31:18.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:18 smithi019 ceph-mon[29267]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:31:18.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:18 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1972144434' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:31:18.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:18 smithi019 ceph-mon[24574]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:31:19.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:18 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1972144434' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:31:19.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:18 smithi172 ceph-mon[25457]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:31:20.657 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:20 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:31:20.501Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=6 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": dial tcp 172.21.15.172:8443: connect: connection refused" 2024-02-07T12:31:20.657 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:20 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/935019539' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-02-07T12:31:20.657 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:20 smithi019 ceph-mon[24574]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-02-07T12:31:20.657 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:20 smithi019 ceph-mon[24574]: osdmap e94: 8 total, 8 up, 8 in 2024-02-07T12:31:20.658 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:20 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:31:20.401+0000 7f9cd5647700 -1 mgr handle_mgr_map I was active but no longer amignoring --setuser ceph since I am not root 2024-02-07T12:31:20.658 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:20 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ignoring --setgroup ceph since I am not root 2024-02-07T12:31:20.658 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:20 smithi019 ceph-mgr[24810]: -- 172.21.15.19:0/3710208587 <== mon.1 v2:172.21.15.19:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559e3b730340 con 0x559e3b708c00 2024-02-07T12:31:20.658 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:20 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/935019539' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-02-07T12:31:20.658 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:20 smithi019 ceph-mon[29267]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-02-07T12:31:20.658 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:20 smithi019 ceph-mon[29267]: osdmap e94: 8 total, 8 up, 8 in 2024-02-07T12:31:20.665 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:20 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/935019539' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-02-07T12:31:20.665 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:20 smithi172 ceph-mon[25457]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-02-07T12:31:20.665 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:20 smithi172 ceph-mon[25457]: osdmap e94: 8 total, 8 up, 8 in 2024-02-07T12:31:20.665 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:31:20 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:31:20] ENGINE Bus STOPPING 2024-02-07T12:31:20.666 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:31:20 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:31:20] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-02-07T12:31:20.666 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:31:20 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:31:20] ENGINE Bus STOPPED 2024-02-07T12:31:20.666 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:31:20 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:31:20] ENGINE Bus STARTING 2024-02-07T12:31:20.666 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:31:20 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:31:20] ENGINE Serving on http://:::9283 2024-02-07T12:31:20.666 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:31:20 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:31:20] ENGINE Bus STARTED 2024-02-07T12:31:21.110 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'sleep 180' 2024-02-07T12:31:21.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:20 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:31:20.945+0000 7f305b4cd000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-02-07T12:31:21.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:31:21.039+0000 7f305b4cd000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-02-07T12:31:21.680 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-02-07T12:31:21.680 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: mgrmap e23: x(active, starting, since 0.0623489s) 2024-02-07T12:31:21.680 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:31:21.680 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:31:21.680 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:31:21.680 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-02-07T12:31:21.681 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:31:21.681 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:31:21.681 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:31:21.681 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:31:21.681 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:31:21.681 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:31:21.681 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:31:21.681 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:31:21.681 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:31:21.681 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:31:21.681 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:31:21.681 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: Manager daemon x is now available 2024-02-07T12:31:21.681 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:21.681 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: Queued rgw.foo for migration 2024-02-07T12:31:21.681 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-02-07T12:31:21.682 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:21.682 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:31:21.682 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:31:21.682 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:31:21.682 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:31:21.682 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:31:21.683 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-02-07T12:31:21.683 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-02-07T12:31:21.683 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:21.683 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:21.683 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: Deploying cephadm binary to smithi019 2024-02-07T12:31:21.683 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[29267]: Deploying cephadm binary to smithi172 2024-02-07T12:31:21.683 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-02-07T12:31:21.684 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: mgrmap e23: x(active, starting, since 0.0623489s) 2024-02-07T12:31:21.684 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:31:21.684 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:31:21.684 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:31:21.684 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-02-07T12:31:21.684 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:31:21.684 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:31:21.684 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:31:21.684 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:31:21.684 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:31:21.684 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:31:21.684 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:31:21.684 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:31:21.684 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:31:21.685 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:31:21.685 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:31:21.685 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: Manager daemon x is now available 2024-02-07T12:31:21.685 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:21.685 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: Queued rgw.foo for migration 2024-02-07T12:31:21.685 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-02-07T12:31:21.685 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:21.685 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:31:21.685 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:31:21.685 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:31:21.685 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:31:21.685 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:31:21.685 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-02-07T12:31:21.685 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-02-07T12:31:21.685 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:21.686 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:21.686 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: Deploying cephadm binary to smithi019 2024-02-07T12:31:21.686 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-mon[24574]: Deploying cephadm binary to smithi172 2024-02-07T12:31:21.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-02-07T12:31:21.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: mgrmap e23: x(active, starting, since 0.0623489s) 2024-02-07T12:31:21.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:31:21.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:31:21.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:31:21.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-02-07T12:31:21.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:31:21.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:31:21.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:31:21.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:31:21.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:31:21.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:31:21.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:31:21.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:31:21.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:31:21.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:31:21.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:31:21.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: Manager daemon x is now available 2024-02-07T12:31:21.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:21.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: Queued rgw.foo for migration 2024-02-07T12:31:21.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-02-07T12:31:21.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:21.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:31:21.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:31:21.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:31:21.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:31:21.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:31:21.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-02-07T12:31:21.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-02-07T12:31:21.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:21.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:21.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: Deploying cephadm binary to smithi019 2024-02-07T12:31:21.872 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:21 smithi172 ceph-mon[25457]: Deploying cephadm binary to smithi172 2024-02-07T12:31:22.015 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:31:21.914Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=7 err="Post \"https://172.21.15.19:8443//api/prometheus_receiver\": dial tcp 172.21.15.19:8443: connect: connection refused" 2024-02-07T12:31:22.015 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:31:21.677+0000 7f305b4cd000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-02-07T12:31:22.016 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:31:21.773+0000 7f305b4cd000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-02-07T12:31:22.016 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:21 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:31:21.860+0000 7f305b4cd000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-02-07T12:31:22.016 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:22 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:31:22.013+0000 7f305b4cd000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-02-07T12:31:22.300 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:22 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:31:22.098+0000 7f305b4cd000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-02-07T12:31:22.301 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:22 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:31:22.212+0000 7f305b4cd000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-02-07T12:31:22.574 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:22 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:31:22.298+0000 7f305b4cd000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-02-07T12:31:22.574 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:22 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:31:22.469+0000 7f305b4cd000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-02-07T12:31:22.574 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:22 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:31:22.572+0000 7f305b4cd000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-02-07T12:31:22.574 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:22 smithi019 ceph-mon[29267]: mgrmap e24: x(active, since 1.07639s) 2024-02-07T12:31:22.575 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:22 smithi019 ceph-mon[29267]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:31:22.575 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:22 smithi019 ceph-mon[29267]: [07/Feb/2024:12:31:21] ENGINE Bus STARTING 2024-02-07T12:31:22.575 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:22 smithi019 ceph-mon[29267]: [07/Feb/2024:12:31:21] ENGINE Serving on http://172.21.15.172:8765 2024-02-07T12:31:22.575 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:22 smithi019 ceph-mon[29267]: [07/Feb/2024:12:31:21] ENGINE Serving on https://172.21.15.172:7150 2024-02-07T12:31:22.575 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:22 smithi019 ceph-mon[29267]: [07/Feb/2024:12:31:21] ENGINE Bus STARTED 2024-02-07T12:31:22.575 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:22 smithi019 ceph-mon[24574]: mgrmap e24: x(active, since 1.07639s) 2024-02-07T12:31:22.575 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:22 smithi019 ceph-mon[24574]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:31:22.575 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:22 smithi019 ceph-mon[24574]: [07/Feb/2024:12:31:21] ENGINE Bus STARTING 2024-02-07T12:31:22.575 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:22 smithi019 ceph-mon[24574]: [07/Feb/2024:12:31:21] ENGINE Serving on http://172.21.15.172:8765 2024-02-07T12:31:22.575 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:22 smithi019 ceph-mon[24574]: [07/Feb/2024:12:31:21] ENGINE Serving on https://172.21.15.172:7150 2024-02-07T12:31:22.575 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:22 smithi019 ceph-mon[24574]: [07/Feb/2024:12:31:21] ENGINE Bus STARTED 2024-02-07T12:31:22.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:22 smithi172 ceph-mon[25457]: mgrmap e24: x(active, since 1.07639s) 2024-02-07T12:31:22.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:22 smithi172 ceph-mon[25457]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:31:22.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:22 smithi172 ceph-mon[25457]: [07/Feb/2024:12:31:21] ENGINE Bus STARTING 2024-02-07T12:31:22.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:22 smithi172 ceph-mon[25457]: [07/Feb/2024:12:31:21] ENGINE Serving on http://172.21.15.172:8765 2024-02-07T12:31:22.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:22 smithi172 ceph-mon[25457]: [07/Feb/2024:12:31:21] ENGINE Serving on https://172.21.15.172:7150 2024-02-07T12:31:22.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:22 smithi172 ceph-mon[25457]: [07/Feb/2024:12:31:21] ENGINE Bus STARTED 2024-02-07T12:31:23.772 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:31:23.636+0000 7f305b4cd000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-02-07T12:31:23.773 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:23 smithi019 ceph-mon[29267]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:31:23.773 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:23 smithi019 ceph-mon[29267]: mgrmap e25: x(active, since 2s) 2024-02-07T12:31:23.773 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:23 smithi019 ceph-mon[24574]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:31:23.773 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:23 smithi019 ceph-mon[24574]: mgrmap e25: x(active, since 2s) 2024-02-07T12:31:23.773 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:31:23.483Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=7 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:31:23.773 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=error ts=2024-02-07T12:31:23.520Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.19:8443//api/prometheus_receiver\": dial tcp 172.21.15.19:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:31:23.774 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:31:23.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.19:8443//api/prometheus_receiver\": dial tcp 172.21.15.19:8443: connect: connection refused" 2024-02-07T12:31:23.774 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:31:23.532Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:31:23.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:23 smithi172 ceph-mon[25457]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:31:23.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:23 smithi172 ceph-mon[25457]: mgrmap e25: x(active, since 2s) 2024-02-07T12:31:24.060 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:31:23.770+0000 7f305b4cd000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-02-07T12:31:24.060 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:31:23.968+0000 7f305b4cd000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-02-07T12:31:24.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:24 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:31:24.058+0000 7f305b4cd000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-02-07T12:31:24.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:24 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:31:24.199+0000 7f305b4cd000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:31:24.762 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:24 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:31:24.763 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:24 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:31:24.763 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:24 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:31:24.529+0000 7f305b4cd000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-02-07T12:31:24.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:24 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:31:25.033 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:24 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:31:24.760+0000 7f305b4cd000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:31:25.033 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:24 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:31:24.848+0000 7f305b4cd000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-02-07T12:31:25.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:25 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:31:25.031+0000 7f305b4cd000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-02-07T12:31:25.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:25 smithi019 ceph-mon[24574]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:31:25.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:25 smithi019 ceph-mon[24574]: mgrmap e26: x(active, since 4s) 2024-02-07T12:31:25.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:25 smithi019 ceph-mon[29267]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:31:25.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:25 smithi019 ceph-mon[29267]: mgrmap e26: x(active, since 4s) 2024-02-07T12:31:25.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:25 smithi172 ceph-mon[25457]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:31:25.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:25 smithi172 ceph-mon[25457]: mgrmap e26: x(active, since 4s) 2024-02-07T12:31:26.330 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:26 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:31:26.000+0000 7f305b4cd000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-02-07T12:31:26.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:26 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:31:26.328+0000 7f305b4cd000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-02-07T12:31:26.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:26 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:31:26.432+0000 7f305b4cd000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-02-07T12:31:27.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:26 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:31:26.598+0000 7f305b4cd000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-02-07T12:31:27.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:26 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: [07/Feb/2024:12:31:26] ENGINE Bus STARTING 2024-02-07T12:31:27.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:26 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: CherryPy Checker: 2024-02-07T12:31:27.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:26 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: The Application mounted at '' has an empty config. 2024-02-07T12:31:27.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:26 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: 2024-02-07T12:31:27.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:26 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: [07/Feb/2024:12:31:26] ENGINE Serving on http://:::9283 2024-02-07T12:31:27.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:26 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: [07/Feb/2024:12:31:26] ENGINE Bus STARTED 2024-02-07T12:31:27.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:27 smithi019 ceph-mon[29267]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:31:27.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:27 smithi019 ceph-mon[29267]: Standby manager daemon y started 2024-02-07T12:31:27.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:27 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.19:0/1029193199' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-02-07T12:31:27.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:27 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.19:0/1029193199' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:31:27.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:27 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.19:0/1029193199' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-02-07T12:31:27.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:27 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.19:0/1029193199' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:31:27.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:27 smithi019 ceph-mon[24574]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:31:27.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:27 smithi019 ceph-mon[24574]: Standby manager daemon y started 2024-02-07T12:31:27.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:27 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.19:0/1029193199' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-02-07T12:31:27.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:27 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.19:0/1029193199' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:31:27.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:27 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.19:0/1029193199' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-02-07T12:31:27.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:27 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.19:0/1029193199' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:31:27.810 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:27 smithi172 ceph-mon[25457]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:31:27.810 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:27 smithi172 ceph-mon[25457]: Standby manager daemon y started 2024-02-07T12:31:27.810 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:27 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.19:0/1029193199' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-02-07T12:31:27.810 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:27 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.19:0/1029193199' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:31:27.810 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:27 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.19:0/1029193199' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-02-07T12:31:27.810 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:27 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.19:0/1029193199' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:31:28.119 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:31:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:31:27] "GET /metrics HTTP/1.1" 200 34898 "" "Prometheus/2.33.4" 2024-02-07T12:31:28.263 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:27 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:31:27.949Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=5 err="Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs" 2024-02-07T12:31:28.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:28 smithi019 ceph-mon[24574]: mgrmap e27: x(active, since 7s), standbys: y 2024-02-07T12:31:28.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:28 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-02-07T12:31:28.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:28 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:31:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:31:28.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:28 smithi019 ceph-mon[29267]: mgrmap e27: x(active, since 7s), standbys: y 2024-02-07T12:31:28.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:28 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-02-07T12:31:28.715 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:28 smithi172 ceph-mon[25457]: mgrmap e27: x(active, since 7s), standbys: y 2024-02-07T12:31:28.715 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:28 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-02-07T12:31:30.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:30 smithi019 ceph-mon[24574]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-02-07T12:31:30.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:30 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:30.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:30 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:30.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:30 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:30.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:30 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:30.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:30 smithi019 ceph-mon[29267]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-02-07T12:31:30.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:30 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:30.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:30 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:30.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:30 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:30.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:30 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:30.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:30 smithi172 ceph-mon[25457]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-02-07T12:31:30.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:30 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:30.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:30 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:30.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:30 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:30.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:30 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:31.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:31.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:31.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: Adjusting osd_memory_target on smithi019 to 2993M 2024-02-07T12:31:31.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:31.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-02-07T12:31:31.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:31.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:31.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: Adjusting osd_memory_target on smithi172 to 3248M 2024-02-07T12:31:31.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:31.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:31:31.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:31:31.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: Updating smithi019:/etc/ceph/ceph.conf 2024-02-07T12:31:31.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: Updating smithi172:/etc/ceph/ceph.conf 2024-02-07T12:31:31.597 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:31.597 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:31.597 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.597 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.597 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.597 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.597 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.597 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.597 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.597 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.597 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: Adjusting osd_memory_target on smithi019 to 2993M 2024-02-07T12:31:31.597 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:31.597 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-02-07T12:31:31.598 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:31.598 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:31.598 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.598 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.598 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.598 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.598 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.598 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.598 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.598 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.598 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: Adjusting osd_memory_target on smithi172 to 3248M 2024-02-07T12:31:31.598 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:31.598 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:31:31.598 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:31:31.598 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: Updating smithi019:/etc/ceph/ceph.conf 2024-02-07T12:31:31.598 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: Updating smithi172:/etc/ceph/ceph.conf 2024-02-07T12:31:31.599 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:31:31.599 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:31:31.599 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[29267]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:31:31.599 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:31:31.599 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:31:31.599 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:31 smithi019 ceph-mon[24574]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:31:31.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:31.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:31.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: Adjusting osd_memory_target on smithi019 to 2993M 2024-02-07T12:31:31.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:31.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-02-07T12:31:31.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:31.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:31.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:31:31.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: Adjusting osd_memory_target on smithi172 to 3248M 2024-02-07T12:31:31.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:31.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:31:31.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:31:31.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: Updating smithi019:/etc/ceph/ceph.conf 2024-02-07T12:31:31.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: Updating smithi172:/etc/ceph/ceph.conf 2024-02-07T12:31:31.622 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:31:31.622 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:31:31.622 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:31 smithi172 ceph-mon[25457]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:31:32.594 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:32 smithi019 systemd[1]: Stopping Ceph node-exporter.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:31:32.594 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:32 smithi019 bash[58173]: Error: no container with name or ID "ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter.a" found: no such container 2024-02-07T12:31:32.595 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:32 smithi019 podman[58186]: 2024-02-07 12:31:32.449208909 +0000 UTC m=+0.140352613 container died de9599dbfeff9861b6e6afb015e173ba658c5219e6c13d1aec84f6bd7272dbb7 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-02-07T12:31:33.041 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:32 smithi019 podman[58186]: 2024-02-07 12:31:32.78870179 +0000 UTC m=+0.479845495 container cleanup de9599dbfeff9861b6e6afb015e173ba658c5219e6c13d1aec84f6bd7272dbb7 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-02-07T12:31:33.041 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:32 smithi019 bash[58186]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a 2024-02-07T12:31:33.041 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:32 smithi019 bash[58212]: Error: no container with name or ID "ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter.a" found: no such container 2024-02-07T12:31:33.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[24574]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:31:33.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[24574]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:31:33.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[24574]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:31:33.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:33.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:33.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:33.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:33.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:33.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:33.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:33.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:33.043 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:33.043 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[24574]: Reconfiguring node-exporter.a (dependencies changed)... 2024-02-07T12:31:33.043 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[24574]: Reconfiguring daemon node-exporter.a on smithi019 2024-02-07T12:31:33.043 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[29267]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:31:33.043 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[29267]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:31:33.043 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[29267]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:31:33.043 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:33.043 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:33.043 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:33.043 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:33.043 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:33.043 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:33.043 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:33.044 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:33.044 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:33.044 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[29267]: Reconfiguring node-exporter.a (dependencies changed)... 2024-02-07T12:31:33.044 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:32 smithi019 ceph-mon[29267]: Reconfiguring daemon node-exporter.a on smithi019 2024-02-07T12:31:33.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:32 smithi172 ceph-mon[25457]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:31:33.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:32 smithi172 ceph-mon[25457]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:31:33.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:32 smithi172 ceph-mon[25457]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:31:33.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:32 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:33.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:32 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:33.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:32 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:33.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:32 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:33.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:32 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:33.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:32 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:33.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:32 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:33.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:32 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:33.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:32 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:33.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:32 smithi172 ceph-mon[25457]: Reconfiguring node-exporter.a (dependencies changed)... 2024-02-07T12:31:33.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:32 smithi172 ceph-mon[25457]: Reconfiguring daemon node-exporter.a on smithi019 2024-02-07T12:31:33.344 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 podman[58200]: 2024-02-07 12:31:33.039980946 +0000 UTC m=+0.595315273 container remove de9599dbfeff9861b6e6afb015e173ba658c5219e6c13d1aec84f6bd7272dbb7 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-02-07T12:31:33.344 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-02-07T12:31:33.344 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@node-exporter.a.service: Failed with result 'exit-code'. 2024-02-07T12:31:33.344 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 systemd[1]: Stopped Ceph node-exporter.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:31:33.345 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@node-exporter.a.service: Consumed 3.725s CPU time. 2024-02-07T12:31:33.345 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 systemd[1]: Starting Ceph node-exporter.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:31:33.345 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 podman[58282]: 2024-02-07 12:31:33.170526174 +0000 UTC m=+0.017875322 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-02-07T12:31:33.345 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 podman[58282]: 2024-02-07 12:31:33.274230208 +0000 UTC m=+0.121579411 container create 9868df14bcc98e518765be8ca65eb64df8aa96b470d7567de6ec2e56ee7421e9 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-02-07T12:31:33.844 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=error ts=2024-02-07T12:31:33.520Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs" 2024-02-07T12:31:33.844 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:31:33.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.19:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.19 because it doesn't contain any IP SANs" 2024-02-07T12:31:33.844 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=warn ts=2024-02-07T12:31:33.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-02-07T12:31:33.845 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 podman[58282]: 2024-02-07 12:31:33.484722616 +0000 UTC m=+0.332071768 container init 9868df14bcc98e518765be8ca65eb64df8aa96b470d7567de6ec2e56ee7421e9 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-02-07T12:31:33.845 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 podman[58282]: 2024-02-07 12:31:33.487370946 +0000 UTC m=+0.334720085 container start 9868df14bcc98e518765be8ca65eb64df8aa96b470d7567de6ec2e56ee7421e9 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-02-07T12:31:33.845 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.490Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-02-07T12:31:33.845 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.490Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-02-07T12:31:33.845 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-02-07T12:31:33.845 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-02-07T12:31:33.845 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-02-07T12:31:33.845 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=node_exporter.go:115 level=info collector=arp 2024-02-07T12:31:33.845 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=node_exporter.go:115 level=info collector=bcache 2024-02-07T12:31:33.846 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=node_exporter.go:115 level=info collector=bonding 2024-02-07T12:31:33.846 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=node_exporter.go:115 level=info collector=btrfs 2024-02-07T12:31:33.846 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=node_exporter.go:115 level=info collector=conntrack 2024-02-07T12:31:33.846 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=node_exporter.go:115 level=info collector=cpu 2024-02-07T12:31:33.846 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-02-07T12:31:33.846 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=node_exporter.go:115 level=info collector=diskstats 2024-02-07T12:31:33.846 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=node_exporter.go:115 level=info collector=dmi 2024-02-07T12:31:33.846 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=node_exporter.go:115 level=info collector=edac 2024-02-07T12:31:33.846 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=node_exporter.go:115 level=info collector=entropy 2024-02-07T12:31:33.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-02-07T12:31:33.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=node_exporter.go:115 level=info collector=filefd 2024-02-07T12:31:33.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=node_exporter.go:115 level=info collector=filesystem 2024-02-07T12:31:33.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=node_exporter.go:115 level=info collector=hwmon 2024-02-07T12:31:33.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=node_exporter.go:115 level=info collector=infiniband 2024-02-07T12:31:33.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=node_exporter.go:115 level=info collector=ipvs 2024-02-07T12:31:33.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=node_exporter.go:115 level=info collector=loadavg 2024-02-07T12:31:33.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=node_exporter.go:115 level=info collector=mdadm 2024-02-07T12:31:33.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=node_exporter.go:115 level=info collector=meminfo 2024-02-07T12:31:33.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=node_exporter.go:115 level=info collector=netclass 2024-02-07T12:31:33.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=node_exporter.go:115 level=info collector=netdev 2024-02-07T12:31:33.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=node_exporter.go:115 level=info collector=netstat 2024-02-07T12:31:33.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=node_exporter.go:115 level=info collector=nfs 2024-02-07T12:31:33.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.491Z caller=node_exporter.go:115 level=info collector=nfsd 2024-02-07T12:31:33.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.492Z caller=node_exporter.go:115 level=info collector=nvme 2024-02-07T12:31:33.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.492Z caller=node_exporter.go:115 level=info collector=os 2024-02-07T12:31:33.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.492Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-02-07T12:31:33.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.492Z caller=node_exporter.go:115 level=info collector=pressure 2024-02-07T12:31:33.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.492Z caller=node_exporter.go:115 level=info collector=rapl 2024-02-07T12:31:33.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.492Z caller=node_exporter.go:115 level=info collector=schedstat 2024-02-07T12:31:33.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.492Z caller=node_exporter.go:115 level=info collector=sockstat 2024-02-07T12:31:33.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.492Z caller=node_exporter.go:115 level=info collector=softnet 2024-02-07T12:31:33.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.492Z caller=node_exporter.go:115 level=info collector=stat 2024-02-07T12:31:33.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.492Z caller=node_exporter.go:115 level=info collector=tapestats 2024-02-07T12:31:33.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.492Z caller=node_exporter.go:115 level=info collector=textfile 2024-02-07T12:31:33.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.492Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-02-07T12:31:33.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.492Z caller=node_exporter.go:115 level=info collector=time 2024-02-07T12:31:33.849 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.492Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-02-07T12:31:33.849 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.492Z caller=node_exporter.go:115 level=info collector=uname 2024-02-07T12:31:33.849 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.492Z caller=node_exporter.go:115 level=info collector=vmstat 2024-02-07T12:31:33.849 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.492Z caller=node_exporter.go:115 level=info collector=xfs 2024-02-07T12:31:33.849 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.492Z caller=node_exporter.go:115 level=info collector=zfs 2024-02-07T12:31:33.849 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.492Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-02-07T12:31:33.849 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[58296]: ts=2024-02-07T12:31:33.492Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-02-07T12:31:33.849 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 bash[58282]: 9868df14bcc98e518765be8ca65eb64df8aa96b470d7567de6ec2e56ee7421e9 2024-02-07T12:31:33.849 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:31:33 smithi019 systemd[1]: Started Ceph node-exporter.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:31:34.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:34 smithi019 ceph-mon[24574]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-02-07T12:31:34.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:34 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:34.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:34 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:34.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:34 smithi019 ceph-mon[29267]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-02-07T12:31:34.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:34 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:34.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:34 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:34 smithi172 ceph-mon[25457]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-02-07T12:31:34.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:34 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:34.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:34 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:35.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:35 smithi019 ceph-mon[24574]: Reconfiguring alertmanager.a (dependencies changed)... 2024-02-07T12:31:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:35 smithi019 ceph-mon[24574]: Reconfiguring daemon alertmanager.a on smithi019 2024-02-07T12:31:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:35 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:31:35.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:35 smithi019 ceph-mon[29267]: Reconfiguring alertmanager.a (dependencies changed)... 2024-02-07T12:31:35.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:35 smithi019 ceph-mon[29267]: Reconfiguring daemon alertmanager.a on smithi019 2024-02-07T12:31:35.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:35 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:31:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:35 smithi172 ceph-mon[25457]: Reconfiguring alertmanager.a (dependencies changed)... 2024-02-07T12:31:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:35 smithi172 ceph-mon[25457]: Reconfiguring daemon alertmanager.a on smithi019 2024-02-07T12:31:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:35 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:31:36.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:36 smithi019 ceph-mon[24574]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-02-07T12:31:36.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:36 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:31:36.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:36 smithi019 ceph-mon[29267]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-02-07T12:31:36.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:36 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:31:36.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:36 smithi172 ceph-mon[25457]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-02-07T12:31:36.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:36 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:31:38.308 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:38 smithi019 ceph-mon[29267]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-02-07T12:31:38.308 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:38 smithi019 ceph-mon[24574]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-02-07T12:31:38.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:38 smithi172 ceph-mon[25457]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-02-07T12:31:38.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:31:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:31:37] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.33.4" 2024-02-07T12:31:38.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:31:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:31:40.085 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:40 smithi019 ceph-mon[29267]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-02-07T12:31:40.085 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:40 smithi019 ceph-mon[24574]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-02-07T12:31:40.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:40 smithi172 ceph-mon[25457]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-02-07T12:31:41.344 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:40 smithi019 systemd[1]: Stopping Ceph alertmanager.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:31:41.345 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:41 smithi019 bash[58562]: Error: no container with name or ID "ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager.a" found: no such container 2024-02-07T12:31:41.345 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[49064]: level=info ts=2024-02-07T12:31:41.124Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-02-07T12:31:41.345 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:41 smithi019 podman[58576]: 2024-02-07 12:31:41.144798781 +0000 UTC m=+0.126196799 container died 4eb1830b3c2021fa79a56d0078a022cb215d66579401a8c62c4771428bb779cf (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-02-07T12:31:41.785 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:41 smithi019 podman[58576]: 2024-02-07 12:31:41.515844492 +0000 UTC m=+0.497242502 container cleanup 4eb1830b3c2021fa79a56d0078a022cb215d66579401a8c62c4771428bb779cf (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-02-07T12:31:41.786 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:41 smithi019 bash[58576]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a 2024-02-07T12:31:41.786 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:41 smithi019 bash[58602]: Error: no container with name or ID "ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager.a" found: no such container 2024-02-07T12:31:42.041 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:41 smithi019 podman[58590]: 2024-02-07 12:31:41.783654691 +0000 UTC m=+0.638790887 container remove 4eb1830b3c2021fa79a56d0078a022cb215d66579401a8c62c4771428bb779cf (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-02-07T12:31:42.041 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:41 smithi019 podman[58590]: 2024-02-07 12:31:41.87598207 +0000 UTC m=+0.731118222 volume remove af4495cbb1d9dd67fbd15f8b3aeabc6305232a0aa10c73b0190d9ead47994e46 2024-02-07T12:31:42.041 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:41 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@alertmanager.a.service: Deactivated successfully. 2024-02-07T12:31:42.041 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:41 smithi019 systemd[1]: Stopped Ceph alertmanager.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:31:42.041 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:41 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@alertmanager.a.service: Consumed 1.033s CPU time. 2024-02-07T12:31:42.041 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:41 smithi019 systemd[1]: Starting Ceph alertmanager.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:31:42.316 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:42 smithi172 ceph-mon[25457]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:31:42.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:42 smithi019 ceph-mon[29267]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:31:42.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:42 smithi019 ceph-mon[24574]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:31:42.348 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:42 smithi019 podman[58672]: 2024-02-07 12:31:42.015677703 +0000 UTC m=+0.018383159 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-02-07T12:31:42.348 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:42 smithi019 podman[58672]: 2024-02-07 12:31:42.12700494 +0000 UTC m=+0.129710441 volume create 63980714c49c66f0d315df879326979f134ed4e9bff9ed0429813c07a76daca1 2024-02-07T12:31:42.348 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:42 smithi019 podman[58672]: 2024-02-07 12:31:42.227328095 +0000 UTC m=+0.230033570 container create dfaf78b29ab547df327fe28e962aae788b67a6c28cecc0e979259756a2f4ceaf (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-02-07T12:31:42.619 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:42 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:31:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: |\n Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed it's capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"172.21.15.19:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.172:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:31:42.844 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:42 smithi019 podman[58672]: 2024-02-07 12:31:42.496392371 +0000 UTC m=+0.499097835 container init dfaf78b29ab547df327fe28e962aae788b67a6c28cecc0e979259756a2f4ceaf (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-02-07T12:31:42.845 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:42 smithi019 podman[58672]: 2024-02-07 12:31:42.499113047 +0000 UTC m=+0.501818502 container start dfaf78b29ab547df327fe28e962aae788b67a6c28cecc0e979259756a2f4ceaf (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-02-07T12:31:42.845 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:42 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[58688]: level=info ts=2024-02-07T12:31:42.510Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-02-07T12:31:42.845 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:42 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[58688]: level=info ts=2024-02-07T12:31:42.510Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-02-07T12:31:42.845 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:42 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[58688]: level=info ts=2024-02-07T12:31:42.511Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.19 port=9094 2024-02-07T12:31:42.845 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:42 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[58688]: level=info ts=2024-02-07T12:31:42.513Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-02-07T12:31:42.845 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:42 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[58688]: level=info ts=2024-02-07T12:31:42.538Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-02-07T12:31:42.845 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:42 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[58688]: level=info ts=2024-02-07T12:31:42.538Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-02-07T12:31:42.845 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:42 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[58688]: level=info ts=2024-02-07T12:31:42.541Z caller=main.go:518 msg=Listening address=:9093 2024-02-07T12:31:42.845 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:42 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[58688]: level=info ts=2024-02-07T12:31:42.541Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-02-07T12:31:42.845 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:42 smithi019 bash[58672]: dfaf78b29ab547df327fe28e962aae788b67a6c28cecc0e979259756a2f4ceaf 2024-02-07T12:31:42.845 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:42 smithi019 systemd[1]: Started Ceph alertmanager.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:31:43.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:43 smithi019 ceph-mon[24574]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:31:43.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:43 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:43.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:43 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:43.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:43 smithi019 ceph-mon[24574]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:31:43.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:43 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:31:43.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:43 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:31:43.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:43 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:31:43.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:43 smithi019 ceph-mon[24574]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:31:43.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:43 smithi019 ceph-mon[29267]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:31:43.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:43 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:43.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:43 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:43.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:43 smithi019 ceph-mon[29267]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:31:43.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:43 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:31:43.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:43 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:31:43.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:43 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:31:43.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:43 smithi019 ceph-mon[29267]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:31:43.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:43 smithi172 ceph-mon[25457]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:31:43.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:43 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:43.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:43 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:43.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:43 smithi172 ceph-mon[25457]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:31:43.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:43 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:31:43.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:43 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:31:43.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:43 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:31:43.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:43 smithi172 ceph-mon[25457]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:31:44.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:44 smithi019 ceph-mon[29267]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:31:44.844 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:44 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[58688]: level=info ts=2024-02-07T12:31:44.514Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.00103974s 2024-02-07T12:31:44.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:44 smithi019 ceph-mon[24574]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:31:45.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:44 smithi172 ceph-mon[25457]: from='client.24700 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:31:45.871 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:45 smithi019 ceph-mon[24574]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:31:45.871 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:45 smithi019 ceph-mon[29267]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:31:46.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:45 smithi172 ceph-mon[25457]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:31:47.992 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:47 smithi172 ceph-mon[25457]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:31:47.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:47 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:47.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:47 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:47.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:47 smithi172 ceph-mon[25457]: Reconfiguring prometheus.a (dependencies changed)... 2024-02-07T12:31:47.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:47 smithi172 ceph-mon[25457]: Reconfiguring daemon prometheus.a on smithi172 2024-02-07T12:31:47.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:47 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/566322351' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-02-07T12:31:47.993 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:47 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3137545924' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1412016607"}]: dispatch 2024-02-07T12:31:48.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:47 smithi019 ceph-mon[24574]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:31:48.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:47 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:48.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:47 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:48.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:47 smithi019 ceph-mon[24574]: Reconfiguring prometheus.a (dependencies changed)... 2024-02-07T12:31:48.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:47 smithi019 ceph-mon[24574]: Reconfiguring daemon prometheus.a on smithi172 2024-02-07T12:31:48.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:47 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/566322351' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-02-07T12:31:48.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:47 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3137545924' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1412016607"}]: dispatch 2024-02-07T12:31:48.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:47 smithi019 ceph-mon[29267]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:31:48.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:47 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:48.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:47 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:48.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:47 smithi019 ceph-mon[29267]: Reconfiguring prometheus.a (dependencies changed)... 2024-02-07T12:31:48.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:47 smithi019 ceph-mon[29267]: Reconfiguring daemon prometheus.a on smithi172 2024-02-07T12:31:48.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:47 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/566322351' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-02-07T12:31:48.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:47 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3137545924' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1412016607"}]: dispatch 2024-02-07T12:31:48.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:31:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:31:47] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.33.4" 2024-02-07T12:31:48.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:31:48 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[24806]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:31:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:31:49.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:48 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3137545924' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1412016607"}]': finished 2024-02-07T12:31:49.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:48 smithi019 ceph-mon[29267]: osdmap e95: 8 total, 8 up, 8 in 2024-02-07T12:31:49.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:48 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3016581879' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/2683363320"}]: dispatch 2024-02-07T12:31:49.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:48 smithi019 ceph-mon[29267]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/2683363320"}]: dispatch 2024-02-07T12:31:49.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:48 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3137545924' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1412016607"}]': finished 2024-02-07T12:31:49.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:48 smithi019 ceph-mon[24574]: osdmap e95: 8 total, 8 up, 8 in 2024-02-07T12:31:49.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:48 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3016581879' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/2683363320"}]: dispatch 2024-02-07T12:31:49.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:48 smithi019 ceph-mon[24574]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/2683363320"}]: dispatch 2024-02-07T12:31:49.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:48 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3137545924' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1412016607"}]': finished 2024-02-07T12:31:49.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:48 smithi172 ceph-mon[25457]: osdmap e95: 8 total, 8 up, 8 in 2024-02-07T12:31:49.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:48 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3016581879' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/2683363320"}]: dispatch 2024-02-07T12:31:49.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:48 smithi172 ceph-mon[25457]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/2683363320"}]: dispatch 2024-02-07T12:31:50.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:49 smithi019 ceph-mon[29267]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-02-07T12:31:50.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:49 smithi019 ceph-mon[29267]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/2683363320"}]': finished 2024-02-07T12:31:50.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:49 smithi019 ceph-mon[29267]: osdmap e96: 8 total, 8 up, 8 in 2024-02-07T12:31:50.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:49 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/824334548' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/2683363320"}]: dispatch 2024-02-07T12:31:50.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:49 smithi019 ceph-mon[29267]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/2683363320"}]: dispatch 2024-02-07T12:31:50.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:49 smithi019 ceph-mon[24574]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-02-07T12:31:50.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:49 smithi019 ceph-mon[24574]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/2683363320"}]': finished 2024-02-07T12:31:50.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:49 smithi019 ceph-mon[24574]: osdmap e96: 8 total, 8 up, 8 in 2024-02-07T12:31:50.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:49 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/824334548' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/2683363320"}]: dispatch 2024-02-07T12:31:50.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:49 smithi019 ceph-mon[24574]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/2683363320"}]: dispatch 2024-02-07T12:31:50.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:49 smithi172 ceph-mon[25457]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-02-07T12:31:50.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:49 smithi172 ceph-mon[25457]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/2683363320"}]': finished 2024-02-07T12:31:50.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:49 smithi172 ceph-mon[25457]: osdmap e96: 8 total, 8 up, 8 in 2024-02-07T12:31:50.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:49 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/824334548' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/2683363320"}]: dispatch 2024-02-07T12:31:50.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:49 smithi172 ceph-mon[25457]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/2683363320"}]: dispatch 2024-02-07T12:31:51.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:50 smithi019 ceph-mon[29267]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/2683363320"}]': finished 2024-02-07T12:31:51.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:50 smithi019 ceph-mon[29267]: osdmap e97: 8 total, 8 up, 8 in 2024-02-07T12:31:51.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:50 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2072547658' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3792014269"}]: dispatch 2024-02-07T12:31:51.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:50 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:31:51.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:50 smithi019 ceph-mon[24574]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/2683363320"}]': finished 2024-02-07T12:31:51.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:50 smithi019 ceph-mon[24574]: osdmap e97: 8 total, 8 up, 8 in 2024-02-07T12:31:51.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:50 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2072547658' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3792014269"}]: dispatch 2024-02-07T12:31:51.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:50 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:31:51.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:50 smithi172 ceph-mon[25457]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/2683363320"}]': finished 2024-02-07T12:31:51.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:50 smithi172 ceph-mon[25457]: osdmap e97: 8 total, 8 up, 8 in 2024-02-07T12:31:51.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:50 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2072547658' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3792014269"}]: dispatch 2024-02-07T12:31:51.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:50 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:31:52.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:51 smithi019 ceph-mon[24574]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-02-07T12:31:52.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:51 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2072547658' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3792014269"}]': finished 2024-02-07T12:31:52.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:51 smithi019 ceph-mon[24574]: osdmap e98: 8 total, 8 up, 8 in 2024-02-07T12:31:52.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:51 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2007782282' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1464010310"}]: dispatch 2024-02-07T12:31:52.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:51 smithi019 ceph-mon[29267]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-02-07T12:31:52.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:51 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2072547658' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3792014269"}]': finished 2024-02-07T12:31:52.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:51 smithi019 ceph-mon[29267]: osdmap e98: 8 total, 8 up, 8 in 2024-02-07T12:31:52.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:51 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2007782282' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1464010310"}]: dispatch 2024-02-07T12:31:52.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:51 smithi172 ceph-mon[25457]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-02-07T12:31:52.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:51 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2072547658' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3792014269"}]': finished 2024-02-07T12:31:52.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:51 smithi172 ceph-mon[25457]: osdmap e98: 8 total, 8 up, 8 in 2024-02-07T12:31:52.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:51 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2007782282' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1464010310"}]: dispatch 2024-02-07T12:31:52.619 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:52 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:31:52.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: |\n Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed it's capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"172.21.15.19:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.172:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:31:52.844 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:31:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[58688]: level=info ts=2024-02-07T12:31:52.515Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002314663s 2024-02-07T12:31:52.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:52 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2007782282' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1464010310"}]': finished 2024-02-07T12:31:52.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:52 smithi019 ceph-mon[29267]: osdmap e99: 8 total, 8 up, 8 in 2024-02-07T12:31:52.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:52 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2905389363' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/479908890"}]: dispatch 2024-02-07T12:31:52.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:52 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2007782282' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1464010310"}]': finished 2024-02-07T12:31:52.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:52 smithi019 ceph-mon[24574]: osdmap e99: 8 total, 8 up, 8 in 2024-02-07T12:31:52.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:52 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2905389363' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/479908890"}]: dispatch 2024-02-07T12:31:53.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:52 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2007782282' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1464010310"}]': finished 2024-02-07T12:31:53.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:52 smithi172 ceph-mon[25457]: osdmap e99: 8 total, 8 up, 8 in 2024-02-07T12:31:53.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:52 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2905389363' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/479908890"}]: dispatch 2024-02-07T12:31:54.055 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:53 smithi019 ceph-mon[29267]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:31:54.055 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:53 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2905389363' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/479908890"}]': finished 2024-02-07T12:31:54.055 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:53 smithi019 ceph-mon[29267]: osdmap e100: 8 total, 8 up, 8 in 2024-02-07T12:31:54.055 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:53 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1662392513' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2799591954"}]: dispatch 2024-02-07T12:31:54.056 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:53 smithi019 ceph-mon[29267]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2799591954"}]: dispatch 2024-02-07T12:31:54.056 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:53 smithi019 ceph-mon[24574]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:31:54.056 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:53 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2905389363' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/479908890"}]': finished 2024-02-07T12:31:54.056 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:53 smithi019 ceph-mon[24574]: osdmap e100: 8 total, 8 up, 8 in 2024-02-07T12:31:54.056 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:53 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1662392513' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2799591954"}]: dispatch 2024-02-07T12:31:54.056 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:53 smithi019 ceph-mon[24574]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2799591954"}]: dispatch 2024-02-07T12:31:54.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:53 smithi172 ceph-mon[25457]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:31:54.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:53 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2905389363' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/479908890"}]': finished 2024-02-07T12:31:54.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:53 smithi172 ceph-mon[25457]: osdmap e100: 8 total, 8 up, 8 in 2024-02-07T12:31:54.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:53 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1662392513' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2799591954"}]: dispatch 2024-02-07T12:31:54.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:53 smithi172 ceph-mon[25457]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2799591954"}]: dispatch 2024-02-07T12:31:55.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:55 smithi019 ceph-mon[24574]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2799591954"}]': finished 2024-02-07T12:31:55.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:55 smithi019 ceph-mon[24574]: osdmap e101: 8 total, 8 up, 8 in 2024-02-07T12:31:55.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:55 smithi019 ceph-mon[29267]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2799591954"}]': finished 2024-02-07T12:31:55.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:55 smithi019 ceph-mon[29267]: osdmap e101: 8 total, 8 up, 8 in 2024-02-07T12:31:55.360 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:55 smithi172 ceph-mon[25457]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2799591954"}]': finished 2024-02-07T12:31:55.360 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:55 smithi172 ceph-mon[25457]: osdmap e101: 8 total, 8 up, 8 in 2024-02-07T12:31:56.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:56 smithi019 ceph-mon[24574]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:31:56.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:56 smithi019 ceph-mon[29267]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:31:56.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:56 smithi172 ceph-mon[25457]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:31:57.027 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:56 smithi172 systemd[1]: Stopping Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:31:57.027 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:56 smithi172 bash[48787]: Error: no container with name or ID "ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus.a" found: no such container 2024-02-07T12:31:57.312 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:31:57.025Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-02-07T12:31:57.312 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:31:57.025Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-02-07T12:31:57.312 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:31:57.025Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-02-07T12:31:57.312 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:31:57.025Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-02-07T12:31:57.312 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:31:57.025Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-02-07T12:31:57.312 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:31:57.025Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-02-07T12:31:57.312 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:31:57.025Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-02-07T12:31:57.312 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:31:57.025Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-02-07T12:31:57.312 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:31:57.025Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-02-07T12:31:57.312 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:31:57.026Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-02-07T12:31:57.312 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:31:57.026Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-02-07T12:31:57.313 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[42348]: ts=2024-02-07T12:31:57.026Z caller=main.go:1066 level=info msg="See you next time!" 2024-02-07T12:31:57.313 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:57 smithi172 podman[48798]: 2024-02-07 12:31:57.035642118 +0000 UTC m=+0.125818830 container died d482181f6c267d79bfe49078ab0879db2c42fd66c34f44b695cfaf633dfa89a8 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:31:57.619 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:57 smithi172 podman[48798]: 2024-02-07 12:31:57.408512658 +0000 UTC m=+0.498689372 container cleanup d482181f6c267d79bfe49078ab0879db2c42fd66c34f44b695cfaf633dfa89a8 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:31:57.619 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:57 smithi172 bash[48798]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a 2024-02-07T12:31:57.619 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:57 smithi172 bash[48824]: Error: no container with name or ID "ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus.a" found: no such container 2024-02-07T12:31:57.980 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:57 smithi172 podman[48812]: 2024-02-07 12:31:57.726447419 +0000 UTC m=+0.692642050 container remove d482181f6c267d79bfe49078ab0879db2c42fd66c34f44b695cfaf633dfa89a8 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:31:57.980 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:57 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@prometheus.a.service: Deactivated successfully. 2024-02-07T12:31:57.980 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:57 smithi172 systemd[1]: Stopped Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:31:57.980 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:57 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@prometheus.a.service: Consumed 1.956s CPU time. 2024-02-07T12:31:57.980 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:57 smithi172 systemd[1]: Starting Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:31:57.980 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:57 smithi172 podman[48891]: 2024-02-07 12:31:57.869566119 +0000 UTC m=+0.018827600 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-02-07T12:31:57.980 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:57 smithi172 podman[48891]: 2024-02-07 12:31:57.977429901 +0000 UTC m=+0.126691423 container create 929fbf32fae22e1d02ece26d158a1699b29c95b22fbfbb22f799213140313054 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:31:58.241 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:58 smithi172 ceph-mon[25457]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 889 B/s rd, 1 op/s 2024-02-07T12:31:58.241 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:58 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:31:58.241 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:58 smithi172 podman[48891]: 2024-02-07 12:31:58.213063674 +0000 UTC m=+0.362325154 container init 929fbf32fae22e1d02ece26d158a1699b29c95b22fbfbb22f799213140313054 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:31:58.241 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:58 smithi172 podman[48891]: 2024-02-07 12:31:58.215659277 +0000 UTC m=+0.364920760 container start 929fbf32fae22e1d02ece26d158a1699b29c95b22fbfbb22f799213140313054 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:31:58.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:58 smithi019 ceph-mon[24574]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 889 B/s rd, 1 op/s 2024-02-07T12:31:58.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:58 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:31:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:58 smithi019 ceph-mon[29267]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 889 B/s rd, 1 op/s 2024-02-07T12:31:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:58 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:31:58.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:31:58.239Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-02-07T12:31:58.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:31:58.239Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-02-07T12:31:58.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:31:58.239Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-02-07T12:31:58.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:31:58.239Z caller=main.go:518 level=info host_details="(Linux 5.14.0-418.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Mon Feb 5 23:17:48 UTC 2024 x86_64 smithi172 (none))" 2024-02-07T12:31:58.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:31:58.239Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-02-07T12:31:58.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:31:58.239Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-02-07T12:31:58.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:31:58.241Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-02-07T12:31:58.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:31:58.241Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-02-07T12:31:58.621 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:31:58.242Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-02-07T12:31:58.621 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:31:58.243Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-02-07T12:31:58.621 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:31:58.243Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.569µs 2024-02-07T12:31:58.621 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:31:58.243Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-02-07T12:31:58.621 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:58 smithi172 bash[48891]: 929fbf32fae22e1d02ece26d158a1699b29c95b22fbfbb22f799213140313054 2024-02-07T12:31:58.621 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:58 smithi172 systemd[1]: Started Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:31:58.621 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:31:58.463Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-02-07T12:31:59.036 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:31:58 smithi172 systemd[1]: Stopping Ceph node-exporter.b for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:31:59.036 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:31:58 smithi172 bash[49016]: Error: no container with name or ID "ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter.b" found: no such container 2024-02-07T12:31:59.037 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:31:58.675Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-02-07T12:31:59.037 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:31:58.675Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-02-07T12:31:59.037 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:31:58.675Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=26.367µs wal_replay_duration=431.842531ms total_replay_duration=431.881687ms 2024-02-07T12:31:59.037 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:31:58.678Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-02-07T12:31:59.037 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:31:58.678Z caller=main.go:947 level=info msg="TSDB started" 2024-02-07T12:31:59.037 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:31:58.678Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-02-07T12:31:59.037 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:31:58.696Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.639819ms db_storage=730ns remote_storage=1.51µs web_handler=333ns query_engine=630ns scrape=170.465µs scrape_sd=94.278µs notify=14.129µs notify_sd=10.09µs rules=17.063224ms 2024-02-07T12:31:59.037 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:31:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:31:58.696Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-02-07T12:31:59.319 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:31:59 smithi172 podman[49030]: 2024-02-07 12:31:59.034576565 +0000 UTC m=+0.107381585 container died 30f1dfd23ff27186b789d921d270bee835dfa0856df3bb586d7db1513482ea82 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-02-07T12:31:59.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:59 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:59.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:59 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:59.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:59 smithi172 ceph-mon[25457]: Reconfiguring node-exporter.b (dependencies changed)... 2024-02-07T12:31:59.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:59 smithi172 ceph-mon[25457]: Reconfiguring daemon node-exporter.b on smithi172 2024-02-07T12:31:59.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:31:59 smithi172 ceph-mon[25457]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 758 B/s rd, 0 op/s 2024-02-07T12:31:59.620 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:31:59 smithi172 podman[49030]: 2024-02-07 12:31:59.407862928 +0000 UTC m=+0.480667946 container cleanup 30f1dfd23ff27186b789d921d270bee835dfa0856df3bb586d7db1513482ea82 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-02-07T12:31:59.620 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:31:59 smithi172 bash[49030]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b 2024-02-07T12:31:59.620 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:31:59 smithi172 bash[49058]: Error: no container with name or ID "ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter.b" found: no such container 2024-02-07T12:31:59.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:59 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:59.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:59 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:59.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:59 smithi019 ceph-mon[29267]: Reconfiguring node-exporter.b (dependencies changed)... 2024-02-07T12:31:59.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:59 smithi019 ceph-mon[29267]: Reconfiguring daemon node-exporter.b on smithi172 2024-02-07T12:31:59.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:31:59 smithi019 ceph-mon[29267]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 758 B/s rd, 0 op/s 2024-02-07T12:31:59.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:59 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:59.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:59 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:31:59.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:59 smithi019 ceph-mon[24574]: Reconfiguring node-exporter.b (dependencies changed)... 2024-02-07T12:31:59.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:59 smithi019 ceph-mon[24574]: Reconfiguring daemon node-exporter.b on smithi172 2024-02-07T12:31:59.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:31:59 smithi019 ceph-mon[24574]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 758 B/s rd, 0 op/s 2024-02-07T12:31:59.905 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:31:59 smithi172 podman[49042]: 2024-02-07 12:31:59.676257285 +0000 UTC m=+0.642839411 container remove 30f1dfd23ff27186b789d921d270bee835dfa0856df3bb586d7db1513482ea82 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-02-07T12:31:59.906 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:31:59 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-02-07T12:31:59.906 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:31:59 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@node-exporter.b.service: Failed with result 'exit-code'. 2024-02-07T12:31:59.906 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:31:59 smithi172 systemd[1]: Stopped Ceph node-exporter.b for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:31:59.906 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:31:59 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@node-exporter.b.service: Consumed 3.711s CPU time. 2024-02-07T12:31:59.906 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:31:59 smithi172 systemd[1]: Starting Ceph node-exporter.b for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:32:00.214 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:31:59 smithi172 podman[49132]: 2024-02-07 12:31:59.808347392 +0000 UTC m=+0.019495492 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-02-07T12:32:00.214 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:31:59 smithi172 podman[49132]: 2024-02-07 12:31:59.927132044 +0000 UTC m=+0.138280182 container create e066f18713209b161990d0825e3d977220e256dc4953a2a54f3ae4fb2f1560c2 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-02-07T12:32:00.214 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 podman[49132]: 2024-02-07 12:32:00.112639801 +0000 UTC m=+0.323787893 container init e066f18713209b161990d0825e3d977220e256dc4953a2a54f3ae4fb2f1560c2 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-02-07T12:32:00.214 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 podman[49132]: 2024-02-07 12:32:00.115390471 +0000 UTC m=+0.326538570 container start e066f18713209b161990d0825e3d977220e256dc4953a2a54f3ae4fb2f1560c2 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-02-07T12:32:00.214 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.118Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-02-07T12:32:00.214 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.118Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-02-07T12:32:00.215 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.119Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-02-07T12:32:00.215 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.119Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-02-07T12:32:00.215 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.119Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-02-07T12:32:00.215 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.119Z caller=node_exporter.go:115 level=info collector=arp 2024-02-07T12:32:00.215 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=bcache 2024-02-07T12:32:00.215 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=bonding 2024-02-07T12:32:00.215 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=btrfs 2024-02-07T12:32:00.215 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=conntrack 2024-02-07T12:32:00.215 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=cpu 2024-02-07T12:32:00.215 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-02-07T12:32:00.216 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=diskstats 2024-02-07T12:32:00.216 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=dmi 2024-02-07T12:32:00.216 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=edac 2024-02-07T12:32:00.216 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=entropy 2024-02-07T12:32:00.216 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-02-07T12:32:00.216 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=filefd 2024-02-07T12:32:00.216 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=filesystem 2024-02-07T12:32:00.216 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=hwmon 2024-02-07T12:32:00.216 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=infiniband 2024-02-07T12:32:00.216 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=ipvs 2024-02-07T12:32:00.216 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=loadavg 2024-02-07T12:32:00.216 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=mdadm 2024-02-07T12:32:00.216 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=meminfo 2024-02-07T12:32:00.217 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=netclass 2024-02-07T12:32:00.217 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=netdev 2024-02-07T12:32:00.217 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=netstat 2024-02-07T12:32:00.217 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=nfs 2024-02-07T12:32:00.217 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=nfsd 2024-02-07T12:32:00.217 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=nvme 2024-02-07T12:32:00.217 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=os 2024-02-07T12:32:00.217 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-02-07T12:32:00.217 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=pressure 2024-02-07T12:32:00.217 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=rapl 2024-02-07T12:32:00.217 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=schedstat 2024-02-07T12:32:00.217 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=sockstat 2024-02-07T12:32:00.217 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=softnet 2024-02-07T12:32:00.217 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=stat 2024-02-07T12:32:00.217 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=tapestats 2024-02-07T12:32:00.218 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=textfile 2024-02-07T12:32:00.218 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-02-07T12:32:00.218 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=time 2024-02-07T12:32:00.218 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-02-07T12:32:00.218 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=uname 2024-02-07T12:32:00.218 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=vmstat 2024-02-07T12:32:00.218 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=xfs 2024-02-07T12:32:00.218 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:115 level=info collector=zfs 2024-02-07T12:32:00.218 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-02-07T12:32:00.218 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[49148]: ts=2024-02-07T12:32:00.120Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-02-07T12:32:00.218 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 bash[49132]: e066f18713209b161990d0825e3d977220e256dc4953a2a54f3ae4fb2f1560c2 2024-02-07T12:32:00.619 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:32:00 smithi172 systemd[1]: Started Ceph node-exporter.b for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:32:01.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:01 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:01.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:01 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:01.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:01 smithi019 ceph-mon[24574]: Reconfiguring grafana.a (dependencies changed)... 2024-02-07T12:32:01.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:01 smithi019 ceph-mon[24574]: Regenerating cephadm self-signed grafana TLS certificates 2024-02-07T12:32:01.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:01 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:01.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:01 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:01.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:01 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-02-07T12:32:01.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:01 smithi019 ceph-mon[24574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-02-07T12:32:01.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:01 smithi019 ceph-mon[24574]: Reconfiguring daemon grafana.a on smithi172 2024-02-07T12:32:01.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:01 smithi019 ceph-mon[24574]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-02-07T12:32:01.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:01 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:01.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:01 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:01.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:01 smithi019 ceph-mon[29267]: Reconfiguring grafana.a (dependencies changed)... 2024-02-07T12:32:01.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:01 smithi019 ceph-mon[29267]: Regenerating cephadm self-signed grafana TLS certificates 2024-02-07T12:32:01.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:01 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:01.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:01 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:01.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:01 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-02-07T12:32:01.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:01 smithi019 ceph-mon[29267]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-02-07T12:32:01.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:01 smithi019 ceph-mon[29267]: Reconfiguring daemon grafana.a on smithi172 2024-02-07T12:32:01.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:01 smithi019 ceph-mon[29267]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-02-07T12:32:01.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:01 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:01.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:01 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:01.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:01 smithi172 ceph-mon[25457]: Reconfiguring grafana.a (dependencies changed)... 2024-02-07T12:32:01.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:01 smithi172 ceph-mon[25457]: Regenerating cephadm self-signed grafana TLS certificates 2024-02-07T12:32:01.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:01 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:01.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:01 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:01.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:01 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-02-07T12:32:01.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:01 smithi172 ceph-mon[25457]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-02-07T12:32:01.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:01 smithi172 ceph-mon[25457]: Reconfiguring daemon grafana.a on smithi172 2024-02-07T12:32:01.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:01 smithi172 ceph-mon[25457]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-02-07T12:32:04.299 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:04 smithi019 ceph-mon[24574]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:32:04.299 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:04 smithi019 ceph-mon[29267]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:32:04.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:04 smithi172 ceph-mon[25457]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:32:06.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:06 smithi019 ceph-mon[29267]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 985 B/s rd, 0 op/s 2024-02-07T12:32:06.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:06 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:32:06.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:06 smithi019 ceph-mon[24574]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 985 B/s rd, 0 op/s 2024-02-07T12:32:06.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:06 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:32:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:06 smithi172 ceph-mon[25457]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 985 B/s rd, 0 op/s 2024-02-07T12:32:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:06 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:32:07.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:32:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:32:07] "GET /metrics HTTP/1.1" 200 37889 "" "Prometheus/2.33.4" 2024-02-07T12:32:08.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:08 smithi019 ceph-mon[29267]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:08.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:08 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:32:08.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:08 smithi019 ceph-mon[24574]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:08.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:08 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:32:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:08 smithi172 ceph-mon[25457]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:08 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:32:09.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:32:08 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:32:08.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:32:10.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:10 smithi019 ceph-mon[24574]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:10.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:10 smithi019 ceph-mon[29267]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:10.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:10 smithi172 ceph-mon[25457]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:12.340 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:12 smithi172 ceph-mon[25457]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:12.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:12 smithi019 ceph-mon[24574]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:12.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:12 smithi019 ceph-mon[29267]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:13.744 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:13 smithi172 systemd[1]: Stopping Ceph grafana.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:32:13.744 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:13 smithi172 bash[49342]: Error: no container with name or ID "ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana.a" found: no such container 2024-02-07T12:32:13.744 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:13 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:32:13+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-02-07T12:32:13.744 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:13 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[41551]: t=2024-02-07T12:32:13+0000 lvl=info msg="Database locked, sleeping then retrying" logger=sqlstore error="database is locked" retry=0 2024-02-07T12:32:13.744 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:13 smithi172 podman[49355]: 2024-02-07 12:32:13.62707532 +0000 UTC m=+0.107776137 container died fcafad07f161817f09ffd319c08bcc4c232dea458f3ae21ee3c6712d789d48be (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a, vcs-type=git, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.24.2, com.redhat.component=ubi8-container, version=8.5, io.openshift.expose-services=, release=236.1648460182, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-03-28T10:36:18.413762, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, description=Ceph Grafana Container, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-02-07T12:32:14.042 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:13 smithi172 podman[49355]: 2024-02-07 12:32:13.939358258 +0000 UTC m=+0.420059075 container cleanup fcafad07f161817f09ffd319c08bcc4c232dea458f3ae21ee3c6712d789d48be (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, io.buildah.version=1.24.2, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, description=Ceph Grafana Container, io.openshift.tags=base rhel8, name=ubi8, build-date=2022-03-28T10:36:18.413762, com.redhat.component=ubi8-container, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, release=236.1648460182, architecture=x86_64) 2024-02-07T12:32:14.043 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:13 smithi172 bash[49355]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a 2024-02-07T12:32:14.043 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:13 smithi172 bash[49384]: Error: no container with name or ID "ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana.a" found: no such container 2024-02-07T12:32:14.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:14 smithi019 ceph-mon[24574]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:14 smithi019 ceph-mon[29267]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:14.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-mon[25457]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:14.370 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 podman[49370]: 2024-02-07 12:32:14.190785666 +0000 UTC m=+0.563746474 container remove fcafad07f161817f09ffd319c08bcc4c232dea458f3ae21ee3c6712d789d48be (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a, build-date=2022-03-28T10:36:18.413762, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.24.2, io.openshift.expose-services=, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, vendor=Red Hat, Inc., name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=236.1648460182, version=8.5, architecture=x86_64, distribution-scope=public) 2024-02-07T12:32:14.370 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@grafana.a.service: Deactivated successfully. 2024-02-07T12:32:14.370 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 systemd[1]: Stopped Ceph grafana.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:32:14.370 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@grafana.a.service: Consumed 3.388s CPU time. 2024-02-07T12:32:14.370 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 systemd[1]: Starting Ceph grafana.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:32:14.687 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 podman[49459]: 2024-02-07 12:32:14.32552503 +0000 UTC m=+0.017989489 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-02-07T12:32:14.687 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 podman[49459]: 2024-02-07 12:32:14.433553085 +0000 UTC m=+0.126017544 container create e753e96fa4f0cabe3a55514250eb56c433a6c35205ae213fe8076ee082a259b4 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a, distribution-scope=public, name=ubi8, io.openshift.expose-services=, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=236.1648460182, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.24.2, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, build-date=2022-03-28T10:36:18.413762) 2024-02-07T12:32:14.687 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 podman[49459]: 2024-02-07 12:32:14.685885447 +0000 UTC m=+0.378349910 container init e753e96fa4f0cabe3a55514250eb56c433a6c35205ae213fe8076ee082a259b4 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, vcs-type=git, io.openshift.tags=base rhel8, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.24.2, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=236.1648460182, vendor=Red Hat, Inc., build-date=2022-03-28T10:36:18.413762, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182) 2024-02-07T12:32:14.981 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:32:14] ENGINE Bus STOPPING 2024-02-07T12:32:14.982 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:32:14] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-02-07T12:32:14.982 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:32:14] ENGINE Bus STOPPED 2024-02-07T12:32:14.982 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:32:14] ENGINE Bus STARTING 2024-02-07T12:32:14.982 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 podman[49459]: 2024-02-07 12:32:14.688732175 +0000 UTC m=+0.381196651 container start e753e96fa4f0cabe3a55514250eb56c433a6c35205ae213fe8076ee082a259b4 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=236.1648460182, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, vcs-type=git, distribution-scope=public, io.buildah.version=1.24.2, build-date=2022-03-28T10:36:18.413762, io.openshift.expose-services=, maintainer=Paul Cuzner , name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., io.openshift.tags=base rhel8) 2024-02-07T12:32:14.982 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-02-07T12:32:14.982 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-02-07T12:32:14.982 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-02-07T12:32:14.983 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-02-07T12:32:14.983 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-02-07T12:32:14.983 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-02-07T12:32:14.983 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-02-07T12:32:14.983 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-02-07T12:32:14.983 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-02-07T12:32:14.983 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-02-07T12:32:14.983 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-02-07T12:32:14.983 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-02-07T12:32:14.984 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-02-07T12:32:14.984 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="App mode production" logger=settings 2024-02-07T12:32:14.984 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-02-07T12:32:14.984 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-02-07T12:32:14.984 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-02-07T12:32:14.984 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=517.838µs 2024-02-07T12:32:14.984 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-02-07T12:32:14.985 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-02-07T12:32:14.985 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-02-07T12:32:14.985 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-02-07T12:32:14.985 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-02-07T12:32:14.985 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-02-07T12:32:14.985 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 bash[49459]: e753e96fa4f0cabe3a55514250eb56c433a6c35205ae213fe8076ee082a259b4 2024-02-07T12:32:14.985 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 systemd[1]: Started Ceph grafana.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:32:14.985 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2024-02-07T12:32:14.985 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-02-07T12:32:14.985 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2024-02-07T12:32:14.985 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-02-07T12:32:14.985 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-02-07T12:32:14.985 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:32:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:32:14+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-02-07T12:32:15.235 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:32:15] ENGINE Serving on http://:::9283 2024-02-07T12:32:15.235 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:32:15] ENGINE Bus STARTED 2024-02-07T12:32:15.235 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:32:15] ENGINE Bus STOPPING 2024-02-07T12:32:15.236 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:32:15] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-02-07T12:32:15.236 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:32:15] ENGINE Bus STOPPED 2024-02-07T12:32:15.236 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:32:15] ENGINE Bus STARTING 2024-02-07T12:32:15.236 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:32:15] ENGINE Serving on http://:::9283 2024-02-07T12:32:15.236 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:32:15] ENGINE Bus STARTED 2024-02-07T12:32:15.236 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:32:15] ENGINE Bus STOPPING 2024-02-07T12:32:15.236 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:32:15] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-02-07T12:32:15.236 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:32:15] ENGINE Bus STOPPED 2024-02-07T12:32:15.236 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:32:15] ENGINE Bus STARTING 2024-02-07T12:32:15.236 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:32:15] ENGINE Serving on http://:::9283 2024-02-07T12:32:15.236 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:32:15] ENGINE Bus STARTED 2024-02-07T12:32:16.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:16.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:16.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:16.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:32:16.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:32:16.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi019.front.sepia.ceph.com:9093"}]: dispatch 2024-02-07T12:32:16.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi019.front.sepia.ceph.com:9093"}]: dispatch 2024-02-07T12:32:16.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:16.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-02-07T12:32:16.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-02-07T12:32:16.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:16.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: Adding iSCSI gateway http://:@172.21.15.19:5000 to Dashboard 2024-02-07T12:32:16.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-02-07T12:32:16.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-02-07T12:32:16.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi019"}]: dispatch 2024-02-07T12:32:16.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi019"}]: dispatch 2024-02-07T12:32:16.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:16.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:32:16.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:32:16.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi172.front.sepia.ceph.com:9095"}]: dispatch 2024-02-07T12:32:16.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi172.front.sepia.ceph.com:9095"}]: dispatch 2024-02-07T12:32:16.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:16.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:32:16.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:32:16.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi172.front.sepia.ceph.com:3000"}]: dispatch 2024-02-07T12:32:16.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi172.front.sepia.ceph.com:3000"}]: dispatch 2024-02-07T12:32:16.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:16.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:32:16.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:16.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:16.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:16.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:32:16.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:32:16.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi019.front.sepia.ceph.com:9093"}]: dispatch 2024-02-07T12:32:16.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi019.front.sepia.ceph.com:9093"}]: dispatch 2024-02-07T12:32:16.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:16.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-02-07T12:32:16.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-02-07T12:32:16.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:16.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: Adding iSCSI gateway http://:@172.21.15.19:5000 to Dashboard 2024-02-07T12:32:16.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-02-07T12:32:16.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-02-07T12:32:16.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi019"}]: dispatch 2024-02-07T12:32:16.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi019"}]: dispatch 2024-02-07T12:32:16.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:16.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:32:16.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:32:16.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi172.front.sepia.ceph.com:9095"}]: dispatch 2024-02-07T12:32:16.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi172.front.sepia.ceph.com:9095"}]: dispatch 2024-02-07T12:32:16.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:16.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:32:16.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:32:16.099 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi172.front.sepia.ceph.com:3000"}]: dispatch 2024-02-07T12:32:16.099 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi172.front.sepia.ceph.com:3000"}]: dispatch 2024-02-07T12:32:16.099 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:16.099 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:15 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:32:16.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:16.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:16.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:16.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:32:16.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:32:16.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi019.front.sepia.ceph.com:9093"}]: dispatch 2024-02-07T12:32:16.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi019.front.sepia.ceph.com:9093"}]: dispatch 2024-02-07T12:32:16.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:16.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-02-07T12:32:16.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-02-07T12:32:16.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:16.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: Adding iSCSI gateway http://:@172.21.15.19:5000 to Dashboard 2024-02-07T12:32:16.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-02-07T12:32:16.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-02-07T12:32:16.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi019"}]: dispatch 2024-02-07T12:32:16.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi019"}]: dispatch 2024-02-07T12:32:16.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:16.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:32:16.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:32:16.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi172.front.sepia.ceph.com:9095"}]: dispatch 2024-02-07T12:32:16.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi172.front.sepia.ceph.com:9095"}]: dispatch 2024-02-07T12:32:16.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:16.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:32:16.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:32:16.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi172.front.sepia.ceph.com:3000"}]: dispatch 2024-02-07T12:32:16.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi172.front.sepia.ceph.com:3000"}]: dispatch 2024-02-07T12:32:16.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:16.122 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:15 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:32:17.572 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:32:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:32:17] "GET /metrics HTTP/1.1" 200 37889 "" "Prometheus/2.33.4" 2024-02-07T12:32:18.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:17 smithi019 ceph-mon[29267]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:18.089 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:17 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:32:18.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:17 smithi019 ceph-mon[24574]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:18.090 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:17 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:32:18.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:17 smithi172 ceph-mon[25457]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:18.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:17 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:32:18.954 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:32:18 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:32:18.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:32:19.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:19 smithi019 ceph-mon[29267]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:19.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:19 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:19.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:19 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:19.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:19 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:19.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:19 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:19.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:19 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:32:19.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:19 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:32:19.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:19 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:19.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:19 smithi019 ceph-mon[24574]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:19.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:19 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:19.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:19 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:19.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:19 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:19.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:19 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:19.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:19 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:32:19.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:19 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:32:19.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:19 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:19.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:19 smithi172 ceph-mon[25457]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:19.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:19 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:19.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:19 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:19.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:19 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:19.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:19 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:19.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:19 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:32:19.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:19 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:32:19.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:19 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:32:20.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:20 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:32:21.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:20 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:32:21.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:20 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:32:21.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:21 smithi172 ceph-mon[25457]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:22.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:21 smithi019 ceph-mon[24574]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:22.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:21 smithi019 ceph-mon[29267]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:24.347 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:24 smithi019 ceph-mon[24574]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:24.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:24 smithi019 ceph-mon[29267]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:24.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:24 smithi172 ceph-mon[25457]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:26.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:26 smithi019 ceph-mon[24574]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:26.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:26 smithi019 ceph-mon[29267]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:26.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:26 smithi172 ceph-mon[25457]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:27.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:32:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:32:27] "GET /metrics HTTP/1.1" 200 37888 "" "Prometheus/2.33.4" 2024-02-07T12:32:28.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:28 smithi019 ceph-mon[24574]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:28.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:28 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:32:28.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:28 smithi019 ceph-mon[29267]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:28.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:28 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:32:28.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:28 smithi172 ceph-mon[25457]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:28.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:28 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:32:29.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:32:28 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:32:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:32:30.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:30 smithi019 ceph-mon[24574]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:30.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:30 smithi019 ceph-mon[29267]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:30.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:30 smithi172 ceph-mon[25457]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:32.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:32 smithi019 ceph-mon[24574]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:32.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:32 smithi019 ceph-mon[29267]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:32.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:32 smithi172 ceph-mon[25457]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:34.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:34 smithi019 ceph-mon[29267]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:34.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:34 smithi019 ceph-mon[24574]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:34 smithi172 ceph-mon[25457]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:36.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:36 smithi019 ceph-mon[24574]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:36.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:36 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:32:36.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:36 smithi019 ceph-mon[29267]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:36.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:36 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:32:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:36 smithi172 ceph-mon[25457]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:36 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:32:37.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:32:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:32:37] "GET /metrics HTTP/1.1" 200 37886 "" "Prometheus/2.33.4" 2024-02-07T12:32:38.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:38 smithi019 ceph-mon[29267]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:38.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:38 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:32:38.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:38 smithi019 ceph-mon[24574]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:38.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:38 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:32:38.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:38 smithi172 ceph-mon[25457]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:38.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:38 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:32:39.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:32:38 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:32:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:32:40.347 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:40 smithi019 ceph-mon[24574]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:40.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:40 smithi019 ceph-mon[29267]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:40.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:40 smithi172 ceph-mon[25457]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:42.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:42 smithi019 ceph-mon[29267]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:42.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:42 smithi019 ceph-mon[24574]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:42.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:42 smithi172 ceph-mon[25457]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:44.341 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:44 smithi172 ceph-mon[25457]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:44.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:44 smithi019 ceph-mon[24574]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:44.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:44 smithi019 ceph-mon[29267]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:46.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:46 smithi019 ceph-mon[29267]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:46.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:46 smithi019 ceph-mon[24574]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:46.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:46 smithi172 ceph-mon[25457]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:47.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:32:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:32:47] "GET /metrics HTTP/1.1" 200 37886 "" "Prometheus/2.33.4" 2024-02-07T12:32:48.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:48 smithi019 ceph-mon[24574]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:48.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:48 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:32:48.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:48 smithi019 ceph-mon[29267]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:48.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:48 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:32:48.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:48 smithi172 ceph-mon[25457]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:48.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:48 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:32:49.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:32:48 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:32:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:32:50.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:50 smithi019 ceph-mon[24574]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:50.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:50 smithi019 ceph-mon[29267]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:50.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:50 smithi172 ceph-mon[25457]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:51.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:51 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:32:51.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:51 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:32:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:51 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:32:52.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:52 smithi019 ceph-mon[24574]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:52.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:52 smithi019 ceph-mon[29267]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:52.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:52 smithi172 ceph-mon[25457]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:54.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:54 smithi019 ceph-mon[24574]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:54.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:54 smithi019 ceph-mon[29267]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:54.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:54 smithi172 ceph-mon[25457]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:32:56.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:56 smithi019 ceph-mon[24574]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:56.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:56 smithi019 ceph-mon[29267]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:56.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:56 smithi172 ceph-mon[25457]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:57.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:32:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:32:57] "GET /metrics HTTP/1.1" 200 37887 "" "Prometheus/2.33.4" 2024-02-07T12:32:58.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:58 smithi019 ceph-mon[24574]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:58.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:32:58 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:32:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:58 smithi019 ceph-mon[29267]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:32:58 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:32:58.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:58 smithi172 ceph-mon[25457]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:32:58.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:32:58 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:32:59.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:32:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:32:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:33:00.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:00 smithi172 ceph-mon[25457]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:00.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:00 smithi019 ceph-mon[24574]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:00.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:00 smithi019 ceph-mon[29267]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:02.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:02 smithi172 ceph-mon[25457]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:02.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:02 smithi019 ceph-mon[24574]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:02.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:02 smithi019 ceph-mon[29267]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:04.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:04 smithi172 ceph-mon[25457]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:04.396 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:04 smithi019 ceph-mon[24574]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:04.397 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:04 smithi019 ceph-mon[29267]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:06 smithi172 ceph-mon[25457]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:06 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:33:06.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:06 smithi019 ceph-mon[24574]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:06.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:06 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:33:06.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:06 smithi019 ceph-mon[29267]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:06.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:06 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:33:07.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:33:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:33:07] "GET /metrics HTTP/1.1" 200 37887 "" "Prometheus/2.33.4" 2024-02-07T12:33:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:08 smithi172 ceph-mon[25457]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:08 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:33:08.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:08 smithi019 ceph-mon[29267]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:08.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:08 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:33:08.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:08 smithi019 ceph-mon[24574]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:08.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:08 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:33:09.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:33:08 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:33:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:33:10.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:10 smithi172 ceph-mon[25457]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:10.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:10 smithi019 ceph-mon[24574]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:10.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:10 smithi019 ceph-mon[29267]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:12.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:12 smithi172 ceph-mon[25457]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:12.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:12 smithi019 ceph-mon[24574]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:12.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:12 smithi019 ceph-mon[29267]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:14.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:14 smithi019 ceph-mon[24574]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:14.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:14 smithi019 ceph-mon[29267]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:14.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:14 smithi172 ceph-mon[25457]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:16.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:16 smithi019 ceph-mon[24574]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:16.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:16 smithi019 ceph-mon[29267]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:16.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:16 smithi172 ceph-mon[25457]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:17.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:33:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:33:17] "GET /metrics HTTP/1.1" 200 37887 "" "Prometheus/2.33.4" 2024-02-07T12:33:18.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:18 smithi019 ceph-mon[29267]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:18.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:18 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:33:18.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:18 smithi019 ceph-mon[24574]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:18.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:18 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:33:18.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:18 smithi172 ceph-mon[25457]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:18.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:18 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:33:19.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:33:18 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:33:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:33:20.476 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:20 smithi019 ceph-mon[29267]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:20.477 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:20 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:33:20.477 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:20 smithi019 ceph-mon[24574]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:20.477 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:20 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:33:20.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:20 smithi172 ceph-mon[25457]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:20.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:20 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:33:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:21 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:33:21.402 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:21 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:33:21.402 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:21 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:33:22.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:22 smithi019 ceph-mon[29267]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:22.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:22 smithi019 ceph-mon[24574]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:22.452 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:22 smithi172 ceph-mon[25457]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:24.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:24 smithi019 ceph-mon[29267]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:24.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:24 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:33:24.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:24 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:33:24.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:24 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:33:24.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:24 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:33:24.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:24 smithi019 ceph-mon[24574]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:24.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:24 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:33:24.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:24 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:33:24.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:24 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:33:24.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:24 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:33:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:24 smithi172 ceph-mon[25457]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:24 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:33:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:24 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:33:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:24 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:33:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:24 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:33:25.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:25 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:33:25.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:25 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:33:25.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:25 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:33:25.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:25 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:33:25.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:25 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:33:25.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:25 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:33:25.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:25 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:33:25.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:25 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:33:25.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:25 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:33:26.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:26 smithi019 ceph-mon[29267]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:26.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:26 smithi019 ceph-mon[24574]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:26.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:26 smithi172 ceph-mon[25457]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:27.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:33:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:33:27] "GET /metrics HTTP/1.1" 200 37888 "" "Prometheus/2.33.4" 2024-02-07T12:33:28.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:28 smithi019 ceph-mon[24574]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:28.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:28 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:33:28.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:28 smithi019 ceph-mon[29267]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:28.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:28 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:33:28.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:28 smithi172 ceph-mon[25457]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:28.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:28 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:33:29.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:33:28 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:33:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:33:30.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:30 smithi019 ceph-mon[29267]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:30.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:30 smithi019 ceph-mon[24574]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:30.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:30 smithi172 ceph-mon[25457]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:32.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:32 smithi019 ceph-mon[29267]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:32.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:32 smithi019 ceph-mon[24574]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:32.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:32 smithi172 ceph-mon[25457]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:34.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:34 smithi019 ceph-mon[24574]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:34.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:34 smithi019 ceph-mon[29267]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:34 smithi172 ceph-mon[25457]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:36.347 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:36 smithi019 ceph-mon[24574]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:36.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:36 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:33:36.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:36 smithi019 ceph-mon[29267]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:36.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:36 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:33:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:36 smithi172 ceph-mon[25457]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:36 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:33:37.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:33:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:33:37] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-02-07T12:33:38.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:38 smithi019 ceph-mon[24574]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:38.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:38 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:33:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:38 smithi019 ceph-mon[29267]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:38 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:33:38.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:38 smithi172 ceph-mon[25457]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:38.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:38 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:33:39.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:33:38 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:33:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:33:40.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:40 smithi019 ceph-mon[29267]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:40.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:40 smithi019 ceph-mon[24574]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:40.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:40 smithi172 ceph-mon[25457]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:42.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:42 smithi019 ceph-mon[29267]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:42 smithi019 ceph-mon[24574]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:42.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:42 smithi172 ceph-mon[25457]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:44.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:44 smithi019 ceph-mon[29267]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:44.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:44 smithi019 ceph-mon[24574]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:44.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:44 smithi172 ceph-mon[25457]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:46.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:46 smithi019 ceph-mon[29267]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:46.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:46 smithi019 ceph-mon[24574]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:46.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:46 smithi172 ceph-mon[25457]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:47.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:33:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:33:47] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-02-07T12:33:48.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:48 smithi019 ceph-mon[29267]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:48.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:48 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:33:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:48 smithi019 ceph-mon[24574]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:48 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:33:48.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:48 smithi172 ceph-mon[25457]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:48.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:48 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:33:49.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:33:48 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:33:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:33:50.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:50 smithi019 ceph-mon[29267]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:50.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:50 smithi019 ceph-mon[24574]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:50.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:50 smithi172 ceph-mon[25457]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:51.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:51 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:33:51.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:51 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:33:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:51 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:33:52.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:52 smithi019 ceph-mon[29267]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:52.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:52 smithi019 ceph-mon[24574]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:52.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:52 smithi172 ceph-mon[25457]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:54.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:54 smithi019 ceph-mon[29267]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:54.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:54 smithi019 ceph-mon[24574]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:54.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:54 smithi172 ceph-mon[25457]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:56.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:56 smithi172 ceph-mon[25457]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:56.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:56 smithi019 ceph-mon[29267]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:56.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:56 smithi019 ceph-mon[24574]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:33:57.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:33:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:33:57] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.33.4" 2024-02-07T12:33:58.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:58 smithi172 ceph-mon[25457]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:58.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:33:58 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:33:58.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:58 smithi019 ceph-mon[29267]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:58.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:33:58 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:33:58.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:58 smithi019 ceph-mon[24574]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:33:58.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:33:58 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:33:59.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:33:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:33:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:34:00.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:00 smithi172 ceph-mon[25457]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:00.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:00 smithi019 ceph-mon[24574]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:00.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:00 smithi019 ceph-mon[29267]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:02.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:02 smithi172 ceph-mon[25457]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:02.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:02 smithi019 ceph-mon[24574]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:02.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:02 smithi019 ceph-mon[29267]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:04.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:04 smithi172 ceph-mon[25457]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:34:04.396 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:04 smithi019 ceph-mon[24574]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:34:04.397 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:04 smithi019 ceph-mon[29267]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:34:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:06 smithi172 ceph-mon[25457]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:06 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:34:06.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:06 smithi019 ceph-mon[24574]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:06.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:06 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:34:06.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:06 smithi019 ceph-mon[29267]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:06.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:06 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:34:07.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:34:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:34:07] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.33.4" 2024-02-07T12:34:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:08 smithi172 ceph-mon[25457]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:34:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:08 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:34:08.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:08 smithi019 ceph-mon[24574]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:34:08.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:08 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:34:08.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:08 smithi019 ceph-mon[29267]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:34:08.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:08 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:34:09.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:34:08 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:34:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:34:10.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:10 smithi172 ceph-mon[25457]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:10.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:10 smithi019 ceph-mon[29267]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:10.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:10 smithi019 ceph-mon[24574]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:12.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:12 smithi019 ceph-mon[24574]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:12.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:12 smithi019 ceph-mon[29267]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:12.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:12 smithi172 ceph-mon[25457]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:14.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:14 smithi019 ceph-mon[24574]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:34:14.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:14 smithi019 ceph-mon[29267]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:34:14.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:14 smithi172 ceph-mon[25457]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:34:16.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:16 smithi019 ceph-mon[29267]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:16.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:16 smithi019 ceph-mon[24574]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:16.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:16 smithi172 ceph-mon[25457]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:17.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:34:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:34:17] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.33.4" 2024-02-07T12:34:18.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:18 smithi019 ceph-mon[29267]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:34:18.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:18 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:34:18.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:18 smithi019 ceph-mon[24574]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:34:18.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:18 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:34:18.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:18 smithi172 ceph-mon[25457]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:34:18.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:18 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:34:19.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:34:18 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:34:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:34:20.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:20 smithi019 ceph-mon[29267]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:20.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:20 smithi019 ceph-mon[24574]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:20.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:20 smithi172 ceph-mon[25457]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:21.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:21 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:34:21.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:21 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:34:21.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:21 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:34:22.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:22 smithi019 ceph-mon[24574]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:22.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:22 smithi019 ceph-mon[29267]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:22.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:22 smithi172 ceph-mon[25457]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:23.580 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-02-07T12:34:24.204 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:24 smithi019 ceph-mon[24574]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:34:24.204 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:24 smithi019 ceph-mon[29267]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:34:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:24 smithi172 ceph-mon[25457]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:34:25.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:25 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:34:25.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:25 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:34:25.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:25 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:34:25.975 INFO:teuthology.orchestra.run.smithi019.stdout:Scheduled to redeploy mgr.y on host 'smithi019' 2024-02-07T12:34:26.234 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:26 smithi019 ceph-mon[24574]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:26.234 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:26 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/164390910' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-02-07T12:34:26.234 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:26 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:26.235 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:26 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:26.235 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:26 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:26.235 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:26 smithi019 ceph-mon[29267]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:26.235 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:26 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/164390910' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-02-07T12:34:26.235 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:26 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:26.235 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:26 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:26.235 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:26 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:26.511 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch ps --refresh' 2024-02-07T12:34:26.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:26 smithi172 ceph-mon[25457]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:26.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:26 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/164390910' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-02-07T12:34:26.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:26 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:26.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:26 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:26.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:26 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:27.507 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:34:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:34:27] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.33.4" 2024-02-07T12:34:27.508 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:27 smithi172 ceph-mon[25457]: from='client.25039 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:34:27.508 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:27 smithi172 ceph-mon[25457]: Schedule redeploy daemon mgr.y 2024-02-07T12:34:27.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:27 smithi019 ceph-mon[29267]: from='client.25039 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:34:27.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:27 smithi019 ceph-mon[29267]: Schedule redeploy daemon mgr.y 2024-02-07T12:34:27.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:27 smithi019 ceph-mon[24574]: from='client.25039 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:34:27.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:27 smithi019 ceph-mon[24574]: Schedule redeploy daemon mgr.y 2024-02-07T12:34:28.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:28 smithi019 ceph-mon[29267]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:34:28.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:28 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:34:28.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:28 smithi019 ceph-mon[24574]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:34:28.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:28 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:34:28.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:28 smithi172 ceph-mon[25457]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:34:28.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:28 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:34:28.372 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:34:28.372 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (2m) 65s ago 10m 17.7M - 0.23.0 ba2b418f427c dfaf78b29ab5 2024-02-07T12:34:28.372 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (2m) 64s ago 10m 42.3M - 8.3.5 dad864ee21e9 e753e96fa4f0 2024-02-07T12:34:28.372 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 running (2m) 65s ago 9m 48.6M - 3.5 e1d6a67b021e 8072de108b37 2024-02-07T12:34:28.372 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283 running (5m) 64s ago 15m 504M - 19.0.0-1252-g44458cce 79957b372307 97644267e529 2024-02-07T12:34:28.372 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:9283 running (16m) 65s ago 16m 411M - 17.2.0 e1d6a67b021e c025f688a668 2024-02-07T12:34:28.372 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (16m) 65s ago 16m 58.2M 2048M 17.2.0 e1d6a67b021e f3a8a050cf1d 2024-02-07T12:34:28.372 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (15m) 64s ago 15m 44.3M 2048M 17.2.0 e1d6a67b021e 7d62c7c4d075 2024-02-07T12:34:28.372 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (15m) 65s ago 15m 48.0M 2048M 17.2.0 e1d6a67b021e b9825d0db32d 2024-02-07T12:34:28.373 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (2m) 65s ago 11m 17.0M - 1.3.1 1dbe0e931976 9868df14bcc9 2024-02-07T12:34:28.373 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (2m) 64s ago 11m 17.1M - 1.3.1 1dbe0e931976 e066f1871320 2024-02-07T12:34:28.373 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (14m) 65s ago 14m 51.8M 2993M 17.2.0 e1d6a67b021e 87f2560a9b15 2024-02-07T12:34:28.373 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (14m) 65s ago 14m 54.0M 2993M 17.2.0 e1d6a67b021e 0f9a190f0e0d 2024-02-07T12:34:28.373 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (13m) 65s ago 13m 50.6M 2993M 17.2.0 e1d6a67b021e 7db94f25a1aa 2024-02-07T12:34:28.373 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (13m) 65s ago 13m 50.9M 2993M 17.2.0 e1d6a67b021e 0b6b674188ed 2024-02-07T12:34:28.373 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (13m) 64s ago 13m 51.2M 3248M 17.2.0 e1d6a67b021e df72cfd69b3b 2024-02-07T12:34:28.373 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (12m) 64s ago 12m 50.1M 3248M 17.2.0 e1d6a67b021e 3bdb22c07537 2024-02-07T12:34:28.373 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (12m) 64s ago 12m 48.1M 3248M 17.2.0 e1d6a67b021e 7859b651e0d9 2024-02-07T12:34:28.373 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (12m) 64s ago 12m 52.6M 3248M 17.2.0 e1d6a67b021e b301e7c75d73 2024-02-07T12:34:28.373 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (2m) 64s ago 11m 62.0M - 2.33.4 514e6a882f6e 929fbf32fae2 2024-02-07T12:34:28.373 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (9m) 65s ago 9m 74.0M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:34:28.373 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (9m) 64s ago 9m 73.3M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:34:29.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:34:28 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:34:28.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:34:29.132 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'sleep 180' 2024-02-07T12:34:29.821 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:29 smithi019 ceph-mon[29267]: from='client.25045 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:34:29.821 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:29 smithi019 ceph-mon[29267]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:29.821 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:29 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:29.821 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:29 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:29.821 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:29 smithi019 ceph-mon[24574]: from='client.25045 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:34:29.821 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:29 smithi019 ceph-mon[24574]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:29.821 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:29 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:29.821 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:29 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:29.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:29 smithi172 ceph-mon[25457]: from='client.25045 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:34:29.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:29 smithi172 ceph-mon[25457]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:29.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:29 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:29.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:29 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:31.757 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:31 smithi019 ceph-mon[29267]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:31.757 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:31 smithi019 ceph-mon[24574]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:31.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:31 smithi172 ceph-mon[25457]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:33.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:33 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:33.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:33 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:33.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:33 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:34:33.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:33 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:34:33.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:33 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:33.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:33 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:34:33.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:33 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:34:33.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:33 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:34:33.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:33 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:34:33.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:33 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:33.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:33 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:33.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:33 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:34:33.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:33 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:34:33.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:33 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:33.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:33 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:34:33.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:33 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:34:33.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:33 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:34:33.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:33 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:34:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:33 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:33 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:33 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:34:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:33 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:34:33.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:33 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:33.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:33 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:34:33.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:33 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:34:33.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:33 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:34:33.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:33 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:34:34.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:34 smithi019 ceph-mon[24574]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:34:34.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:34 smithi019 ceph-mon[24574]: Deploying daemon mgr.y on smithi019 2024-02-07T12:34:34.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:34 smithi019 ceph-mon[29267]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:34:34.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:34 smithi019 ceph-mon[29267]: Deploying daemon mgr.y on smithi019 2024-02-07T12:34:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:34 smithi172 ceph-mon[25457]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:34:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:34 smithi172 ceph-mon[25457]: Deploying daemon mgr.y on smithi019 2024-02-07T12:34:34.976 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:34 smithi019 systemd[1]: Stopping Ceph mgr.y for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:34:35.228 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:34 smithi019 podman[61700]: 2024-02-07 12:34:34.974263949 +0000 UTC m=+0.162201003 container died c025f688a668eca87841aa483a02e4b9f494af3ad340e86323b6df703766a853 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, ceph=True, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , distribution-scope=public, GIT_CLEAN=True, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, name=centos-stream, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754) 2024-02-07T12:34:35.482 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:35 smithi019 podman[61700]: 2024-02-07 12:34:35.309196877 +0000 UTC m=+0.497133925 container cleanup c025f688a668eca87841aa483a02e4b9f494af3ad340e86323b6df703766a853 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , distribution-scope=public, release=754, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, architecture=x86_64, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, version=8, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, io.openshift.expose-services=, com.redhat.component=centos-stream-container) 2024-02-07T12:34:35.482 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:35 smithi019 bash[61700]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y 2024-02-07T12:34:35.482 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:35 smithi019 podman[61714]: 2024-02-07 12:34:35.477236223 +0000 UTC m=+0.501238858 container remove c025f688a668eca87841aa483a02e4b9f494af3ad340e86323b6df703766a853 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, distribution-scope=public, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, io.openshift.expose-services=, maintainer=Guillaume Abrioux , version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, ceph=True, GIT_BRANCH=HEAD) 2024-02-07T12:34:35.482 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:35 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-02-07T12:34:36.083 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:35 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mgr.y.service: Failed with result 'exit-code'. 2024-02-07T12:34:36.084 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:35 smithi019 systemd[1]: Stopped Ceph mgr.y for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:34:36.084 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:35 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mgr.y.service: Consumed 1min 3.060s CPU time. 2024-02-07T12:34:36.084 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:35 smithi019 systemd[1]: Starting Ceph mgr.y for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:34:36.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:36 smithi019 ceph-mon[24574]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:36.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:36 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:34:36.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:36 smithi019 ceph-mon[29267]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:36.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:36 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:34:36.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:36 smithi019 podman[61857]: 2024-02-07 12:34:36.106638575 +0000 UTC m=+0.019027132 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:34:36.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:36 smithi019 podman[61857]: 2024-02-07 12:34:36.211823142 +0000 UTC m=+0.124211692 container create 11bffc8ee7dad2dcb75654595e3891267fe8186f218283242da10338be349402 (image=quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, CEPH_POINT_RELEASE=) 2024-02-07T12:34:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:36 smithi172 ceph-mon[25457]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:36 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:34:36.844 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:36 smithi019 podman[61857]: 2024-02-07 12:34:36.473458581 +0000 UTC m=+0.385847131 container init 11bffc8ee7dad2dcb75654595e3891267fe8186f218283242da10338be349402 (image=quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-02-07T12:34:36.844 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:36 smithi019 podman[61857]: 2024-02-07 12:34:36.47627655 +0000 UTC m=+0.388665099 container start 11bffc8ee7dad2dcb75654595e3891267fe8186f218283242da10338be349402 (image=quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:34:36.844 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:36 smithi019 ceph-mgr[61877]: -- 172.21.15.19:0/499790826 <== mon.0 v2:172.21.15.19:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5580cf581520 con 0x5580cf556c00 2024-02-07T12:34:36.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:36 smithi019 bash[61857]: 11bffc8ee7dad2dcb75654595e3891267fe8186f218283242da10338be349402 2024-02-07T12:34:36.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:36 smithi019 systemd[1]: Started Ceph mgr.y for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:34:37.190 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:34:37.062+0000 7fcd5fde3200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-02-07T12:34:37.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:34:37.187+0000 7fcd5fde3200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-02-07T12:34:37.607 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:34:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:34:37] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.33.4" 2024-02-07T12:34:37.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:37 smithi172 ceph-mon[25457]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-02-07T12:34:37.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:37 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:37.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:37 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:37.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:37 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:37.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:37 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:37.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:37 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:34:37.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:37 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:34:37.903 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:37 smithi019 ceph-mon[24574]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-02-07T12:34:37.903 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:37 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:37.903 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:37 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:37.903 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:37 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:37.903 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:37 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:37.903 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:37 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:34:37.903 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:37 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:34:37.903 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:37 smithi019 ceph-mon[29267]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-02-07T12:34:37.903 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:37 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:37.903 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:37 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:37.904 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:37 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:37.904 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:37 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:37.904 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:37 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:34:37.904 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:37 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:34:38.155 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:34:37.899+0000 7fcd5fde3200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-02-07T12:34:38.155 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:34:38.000+0000 7fcd5fde3200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-02-07T12:34:38.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:34:38.152+0000 7fcd5fde3200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-02-07T12:34:38.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:34:38.252+0000 7fcd5fde3200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-02-07T12:34:38.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:34:38.337+0000 7fcd5fde3200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-02-07T12:34:38.919 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:34:38.608+0000 7fcd5fde3200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-02-07T12:34:38.919 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:34:38.721+0000 7fcd5fde3200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-02-07T12:34:38.919 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:34:38.813+0000 7fcd5fde3200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-02-07T12:34:39.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:34:38 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:34:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:34:39.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:34:38.915+0000 7fcd5fde3200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-02-07T12:34:39.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:34:39.101+0000 7fcd5fde3200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-02-07T12:34:40.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:39 smithi019 ceph-mon[29267]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-02-07T12:34:40.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:39 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:40.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:39 smithi019 ceph-mon[24574]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-02-07T12:34:40.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:39 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:40.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:39 smithi172 ceph-mon[25457]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-02-07T12:34:40.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:39 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:40.441 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:40 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:34:40.158+0000 7fcd5fde3200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-02-07T12:34:40.442 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:40 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:34:40.295+0000 7fcd5fde3200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-02-07T12:34:40.743 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:40 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:34:40.641+0000 7fcd5fde3200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-02-07T12:34:40.743 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:40 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:34:40.739+0000 7fcd5fde3200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-02-07T12:34:41.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:40 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:34:40.976+0000 7fcd5fde3200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:34:41.476 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:34:41.181+0000 7fcd5fde3200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-02-07T12:34:41.762 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:34:41.486+0000 7fcd5fde3200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:34:42.012 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:41 smithi019 ceph-mon[29267]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-02-07T12:34:42.012 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:41 smithi019 ceph-mon[24574]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-02-07T12:34:42.013 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:34:41.758+0000 7fcd5fde3200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-02-07T12:34:42.107 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:41 smithi172 ceph-mon[25457]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-02-07T12:34:42.263 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:42 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:34:42.009+0000 7fcd5fde3200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-02-07T12:34:43.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:42 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:34:42.865+0000 7fcd5fde3200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-02-07T12:34:43.800 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:34:43.466+0000 7fcd5fde3200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-02-07T12:34:43.801 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:34:43.569+0000 7fcd5fde3200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-02-07T12:34:43.801 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:34:43.797+0000 7fcd5fde3200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-02-07T12:34:44.058 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: [07/Feb/2024:12:34:43] ENGINE Bus STARTING 2024-02-07T12:34:44.058 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CherryPy Checker: 2024-02-07T12:34:44.058 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: The Application mounted at '' has an empty config. 2024-02-07T12:34:44.058 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:34:44.058 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: [07/Feb/2024:12:34:43] ENGINE Serving on http://:::9283 2024-02-07T12:34:44.058 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:34:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: [07/Feb/2024:12:34:43] ENGINE Bus STARTED 2024-02-07T12:34:44.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[29267]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-02-07T12:34:44.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:44.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:44.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:44.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:44.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:34:44.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:34:44.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:44.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[29267]: Standby manager daemon y restarted 2024-02-07T12:34:44.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[29267]: Standby manager daemon y started 2024-02-07T12:34:44.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-02-07T12:34:44.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:34:44.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-02-07T12:34:44.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:34:44.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[24574]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-02-07T12:34:44.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:44.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:44.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:44.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:44.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:34:44.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:34:44.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:44.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[24574]: Standby manager daemon y restarted 2024-02-07T12:34:44.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[24574]: Standby manager daemon y started 2024-02-07T12:34:44.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-02-07T12:34:44.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:34:44.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-02-07T12:34:44.347 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:44 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:34:44.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:44 smithi172 ceph-mon[25457]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-02-07T12:34:44.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:44 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:44.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:44 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:44.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:44 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:44.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:44 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:44.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:44 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:34:44.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:44 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:34:44.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:44 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:34:44.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:44 smithi172 ceph-mon[25457]: Standby manager daemon y restarted 2024-02-07T12:34:44.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:44 smithi172 ceph-mon[25457]: Standby manager daemon y started 2024-02-07T12:34:44.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:44 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-02-07T12:34:44.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:44 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:34:44.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:44 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-02-07T12:34:44.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:44 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:34:45.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:45 smithi019 ceph-mon[24574]: mgrmap e28: x(active, since 3m), standbys: y 2024-02-07T12:34:45.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:45 smithi019 ceph-mon[24574]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-02-07T12:34:45.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:45 smithi019 ceph-mon[29267]: mgrmap e28: x(active, since 3m), standbys: y 2024-02-07T12:34:45.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:45 smithi019 ceph-mon[29267]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-02-07T12:34:45.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:45 smithi172 ceph-mon[25457]: mgrmap e28: x(active, since 3m), standbys: y 2024-02-07T12:34:45.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:45 smithi172 ceph-mon[25457]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-02-07T12:34:47.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:34:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:34:47] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.33.4" 2024-02-07T12:34:47.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:47 smithi172 ceph-mon[25457]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-02-07T12:34:47.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:47 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:34:47.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:47 smithi019 ceph-mon[24574]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-02-07T12:34:47.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:47 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:34:47.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:47 smithi019 ceph-mon[29267]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-02-07T12:34:47.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:47 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:34:49.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:34:48 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:34:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:34:50.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:50 smithi019 ceph-mon[24574]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-02-07T12:34:50.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:50 smithi019 ceph-mon[29267]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-02-07T12:34:50.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:50 smithi172 ceph-mon[25457]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-02-07T12:34:51.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:51 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:34:51.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:51 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:34:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:51 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:34:52.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:52 smithi019 ceph-mon[29267]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-02-07T12:34:52.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:52 smithi019 ceph-mon[24574]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-02-07T12:34:52.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:52 smithi172 ceph-mon[25457]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-02-07T12:34:54.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:54 smithi019 ceph-mon[29267]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-02-07T12:34:54.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:54 smithi019 ceph-mon[24574]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-02-07T12:34:54.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:54 smithi172 ceph-mon[25457]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-02-07T12:34:56.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:56 smithi019 ceph-mon[29267]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:56.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:56 smithi019 ceph-mon[24574]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:56.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:56 smithi172 ceph-mon[25457]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:34:57.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:34:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:34:57] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.33.4" 2024-02-07T12:34:58.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:58 smithi019 ceph-mon[29267]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:34:58.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:34:58 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:34:58.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:58 smithi019 ceph-mon[24574]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:34:58.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:34:58 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:34:58.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:58 smithi172 ceph-mon[25457]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:34:58.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:34:58 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:34:59.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:34:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:34:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:35:00.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:00 smithi019 ceph-mon[29267]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:00.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:00 smithi019 ceph-mon[24574]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:00.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:00 smithi172 ceph-mon[25457]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:02.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:02 smithi019 ceph-mon[24574]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:02.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:02 smithi019 ceph-mon[29267]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:02.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:02 smithi172 ceph-mon[25457]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:04.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:04 smithi019 ceph-mon[29267]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:04.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:04 smithi019 ceph-mon[24574]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:04.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:04 smithi172 ceph-mon[25457]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:06 smithi172 ceph-mon[25457]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:06 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:35:06.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:06 smithi019 ceph-mon[24574]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:06.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:06 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:35:06.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:06 smithi019 ceph-mon[29267]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:06.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:06 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:35:07.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:35:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:35:07] "GET /metrics HTTP/1.1" 200 37895 "" "Prometheus/2.33.4" 2024-02-07T12:35:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:08 smithi172 ceph-mon[25457]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:08 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:35:08.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:08 smithi019 ceph-mon[24574]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:08.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:08 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:35:08.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:08 smithi019 ceph-mon[29267]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:08.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:08 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:35:09.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:35:08 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:35:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:35:10.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:10 smithi172 ceph-mon[25457]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:10.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:10 smithi019 ceph-mon[24574]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:10.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:10 smithi019 ceph-mon[29267]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:12.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:12 smithi172 ceph-mon[25457]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:12.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:12 smithi019 ceph-mon[24574]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:12.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:12 smithi019 ceph-mon[29267]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:14.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:14 smithi172 ceph-mon[25457]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:14.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:14 smithi019 ceph-mon[24574]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:14.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:14 smithi019 ceph-mon[29267]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:16.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:16 smithi172 ceph-mon[25457]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:16.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:16 smithi019 ceph-mon[24574]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:16.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:16 smithi019 ceph-mon[29267]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:17.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:35:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:35:17] "GET /metrics HTTP/1.1" 200 37895 "" "Prometheus/2.33.4" 2024-02-07T12:35:18.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:18 smithi019 ceph-mon[29267]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:18.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:18 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:35:18.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:18 smithi019 ceph-mon[24574]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:18.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:18 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:35:18.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:18 smithi172 ceph-mon[25457]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:18.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:18 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:35:19.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:35:18 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:35:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:35:20.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:20 smithi019 ceph-mon[29267]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:20.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:20 smithi019 ceph-mon[24574]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:20.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:20 smithi172 ceph-mon[25457]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:21.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:21 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:35:21.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:21 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:35:21.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:21 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:35:22.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:22 smithi019 ceph-mon[24574]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:22.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:22 smithi019 ceph-mon[29267]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:22.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:22 smithi172 ceph-mon[25457]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:24.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:24 smithi019 ceph-mon[24574]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:24.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:24 smithi019 ceph-mon[29267]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:24 smithi172 ceph-mon[25457]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:26.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:26 smithi019 ceph-mon[24574]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:26.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:26 smithi019 ceph-mon[29267]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:26.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:26 smithi172 ceph-mon[25457]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:27.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:35:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:35:27] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.33.4" 2024-02-07T12:35:28.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:28 smithi019 ceph-mon[24574]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:28.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:28 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:35:28.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:28 smithi019 ceph-mon[29267]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:28.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:28 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:35:28.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:28 smithi172 ceph-mon[25457]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:28.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:28 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:35:29.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:35:28 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:35:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:35:30.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:30 smithi019 ceph-mon[24574]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:30.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:30 smithi019 ceph-mon[29267]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:30.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:30 smithi172 ceph-mon[25457]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:32.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:32 smithi019 ceph-mon[24574]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:32.711 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:32 smithi019 ceph-mon[29267]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:32.712 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:32 smithi172 ceph-mon[25457]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:34.416 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:34 smithi019 ceph-mon[24574]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:34.416 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:34 smithi019 ceph-mon[29267]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:34.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:34 smithi172 ceph-mon[25457]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:36.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:36 smithi019 ceph-mon[24574]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:36.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:36 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:35:36.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:36 smithi019 ceph-mon[29267]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:36.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:36 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:35:36.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:36 smithi172 ceph-mon[25457]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:36.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:36 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:35:37.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:35:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:35:37] "GET /metrics HTTP/1.1" 200 37892 "" "Prometheus/2.33.4" 2024-02-07T12:35:38.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:38 smithi019 ceph-mon[24574]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:38.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:38 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:35:38.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:38 smithi019 ceph-mon[29267]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:38.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:38 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:35:38.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:38 smithi172 ceph-mon[25457]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:38.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:38 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:35:39.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:35:38 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:35:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:35:40.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:40 smithi019 ceph-mon[24574]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:40.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:40 smithi019 ceph-mon[29267]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:40.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:40 smithi172 ceph-mon[25457]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:42.421 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:42 smithi172 ceph-mon[25457]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:42.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:42 smithi019 ceph-mon[24574]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:42.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:42 smithi019 ceph-mon[29267]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:44.517 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:44 smithi019 ceph-mon[24574]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:44.517 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:44 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:35:44.518 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:44 smithi019 ceph-mon[29267]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:44.518 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:44 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:35:44.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:44 smithi172 ceph-mon[25457]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:44.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:44 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:35:46.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:46 smithi019 ceph-mon[24574]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:46.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:46 smithi019 ceph-mon[29267]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:46.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:46 smithi172 ceph-mon[25457]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:47.620 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:35:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:35:47] "GET /metrics HTTP/1.1" 200 37892 "" "Prometheus/2.33.4" 2024-02-07T12:35:48.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:48 smithi019 ceph-mon[24574]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:48.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:48 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:35:48.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:48 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:35:48.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:48 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:35:48.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:48 smithi019 ceph-mon[29267]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:48.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:48 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:35:48.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:48 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:35:48.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:48 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:35:48.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:48 smithi172 ceph-mon[25457]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:48.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:48 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:35:48.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:48 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:35:48.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:48 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:35:49.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:35:48 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:35:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:35:49.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:49 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:35:49.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:49 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:35:49.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:49 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:35:49.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:49 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:35:49.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:49 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:35:49.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:49 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:35:49.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:49 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:35:49.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:49 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:35:49.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:49 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:35:50.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:50 smithi019 ceph-mon[24574]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:50.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:50 smithi019 ceph-mon[29267]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:50.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:50 smithi172 ceph-mon[25457]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:51.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:51 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:35:51.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:51 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:35:51.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:51 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:35:52.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:52 smithi019 ceph-mon[24574]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:52.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:52 smithi019 ceph-mon[29267]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:52.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:52 smithi172 ceph-mon[25457]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:54.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:54 smithi019 ceph-mon[24574]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:54.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:54 smithi019 ceph-mon[29267]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:54.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:54 smithi172 ceph-mon[25457]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:56.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:56 smithi019 ceph-mon[24574]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:56.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:56 smithi019 ceph-mon[29267]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:56.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:56 smithi172 ceph-mon[25457]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:35:57.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:35:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:35:57] "GET /metrics HTTP/1.1" 200 37895 "" "Prometheus/2.33.4" 2024-02-07T12:35:58.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:58 smithi019 ceph-mon[24574]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:58.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:35:58 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:35:58.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:58 smithi019 ceph-mon[29267]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:58.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:35:58 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:35:58.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:58 smithi172 ceph-mon[25457]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:35:58.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:35:58 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:35:59.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:35:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:35:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:36:00.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:00 smithi019 ceph-mon[24574]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:00.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:00 smithi019 ceph-mon[29267]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:00.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:00 smithi172 ceph-mon[25457]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:02.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:02 smithi019 ceph-mon[24574]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:02.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:02 smithi019 ceph-mon[29267]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:02.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:02 smithi172 ceph-mon[25457]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:04.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:04 smithi019 ceph-mon[24574]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:04.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:04 smithi019 ceph-mon[29267]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:04.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:04 smithi172 ceph-mon[25457]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:06.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:06 smithi019 ceph-mon[24574]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:06.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:06 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:36:06.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:06 smithi019 ceph-mon[29267]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:06.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:06 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:36:06.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:06 smithi172 ceph-mon[25457]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:06.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:06 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:36:07.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:36:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:36:07] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.33.4" 2024-02-07T12:36:08.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:08 smithi019 ceph-mon[24574]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:08.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:08 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:36:08.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:08 smithi019 ceph-mon[29267]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:08.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:08 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:36:08.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:08 smithi172 ceph-mon[25457]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:08.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:08 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:36:09.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:36:08 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:36:08.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:36:10.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:10 smithi019 ceph-mon[24574]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:10.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:10 smithi019 ceph-mon[29267]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:10.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:10 smithi172 ceph-mon[25457]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:12.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:12 smithi019 ceph-mon[24574]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:12.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:12 smithi019 ceph-mon[29267]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:12.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:12 smithi172 ceph-mon[25457]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:14.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:14 smithi019 ceph-mon[24574]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:14.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:14 smithi019 ceph-mon[29267]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:14.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:14 smithi172 ceph-mon[25457]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:16.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:16 smithi019 ceph-mon[29267]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:16.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:16 smithi019 ceph-mon[24574]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:16.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:16 smithi172 ceph-mon[25457]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:17.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:36:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:36:17] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.33.4" 2024-02-07T12:36:18.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:18 smithi019 ceph-mon[29267]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:18.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:18 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:36:18.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:18 smithi019 ceph-mon[24574]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:18.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:18 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:36:18.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:18 smithi172 ceph-mon[25457]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:18.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:18 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:36:19.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:36:18 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:36:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:36:20.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:20 smithi019 ceph-mon[29267]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:20.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:20 smithi019 ceph-mon[24574]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:20.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:20 smithi172 ceph-mon[25457]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:21.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:21 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:36:21.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:21 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:36:21.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:21 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:36:22.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:22 smithi019 ceph-mon[24574]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:22.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:22 smithi019 ceph-mon[29267]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:22.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:22 smithi172 ceph-mon[25457]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:24.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:24 smithi019 ceph-mon[24574]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:24.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:24 smithi019 ceph-mon[29267]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:24 smithi172 ceph-mon[25457]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:26.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:26 smithi019 ceph-mon[24574]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:26.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:26 smithi019 ceph-mon[29267]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:26.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:26 smithi172 ceph-mon[25457]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:27.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:36:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:36:27] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.33.4" 2024-02-07T12:36:28.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:28 smithi019 ceph-mon[24574]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:28.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:28 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:36:28.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:28 smithi019 ceph-mon[29267]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:28.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:28 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:36:28.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:28 smithi172 ceph-mon[25457]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:28.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:28 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:36:29.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:36:28 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:36:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:36:30.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:30 smithi019 ceph-mon[24574]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:30.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:30 smithi019 ceph-mon[29267]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:30.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:30 smithi172 ceph-mon[25457]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:32.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:32 smithi019 ceph-mon[24574]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:32.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:32 smithi019 ceph-mon[29267]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:32.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:32 smithi172 ceph-mon[25457]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:34.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:34 smithi019 ceph-mon[24574]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:34.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:34 smithi019 ceph-mon[29267]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:34.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:34 smithi172 ceph-mon[25457]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:36.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:36 smithi019 ceph-mon[24574]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:36.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:36 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:36:36.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:36 smithi019 ceph-mon[29267]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:36.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:36 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:36:36.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:36 smithi172 ceph-mon[25457]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:36.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:36 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:36:37.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:36:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:36:37] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-02-07T12:36:38.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:38 smithi019 ceph-mon[24574]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:38.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:38 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:36:38.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:38 smithi019 ceph-mon[29267]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:38.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:38 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:36:38.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:38 smithi172 ceph-mon[25457]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:38.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:38 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:36:39.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:36:38 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:36:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"smithi172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.19\", device_class=\"ssd\", hostname=\"smithi019\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.19\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-02-07T12:36:40.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:40 smithi019 ceph-mon[24574]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:40.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:40 smithi019 ceph-mon[29267]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:40.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:40 smithi172 ceph-mon[25457]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:42.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:42 smithi019 ceph-mon[24574]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:42.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:42 smithi019 ceph-mon[29267]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:42.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:42 smithi172 ceph-mon[25457]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:44.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:44 smithi019 ceph-mon[24574]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:44.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:44 smithi019 ceph-mon[29267]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:44.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:44 smithi172 ceph-mon[25457]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:46.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:46 smithi019 ceph-mon[24574]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:46.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:46 smithi019 ceph-mon[29267]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:46.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:46 smithi172 ceph-mon[25457]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:47.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:36:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:36:47] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-02-07T12:36:48.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:48 smithi019 ceph-mon[24574]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:48.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:48 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:36:48.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:48 smithi019 ceph-mon[29267]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:48.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:48 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:36:48.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:48 smithi172 ceph-mon[25457]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:48.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:48 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:36:49.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:49 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:36:49.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:49 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:36:49.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:49 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:36:49.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:49 smithi019 ceph-mon[24574]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:36:49.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:49 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:36:49.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:49 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:36:49.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:49 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:36:49.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:49 smithi019 ceph-mon[29267]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:36:49.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:49 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:36:49.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:49 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:36:49.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:49 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:36:49.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:49 smithi172 ceph-mon[25457]: from='mgr.24773 ' entity='mgr.x' 2024-02-07T12:36:50.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:50 smithi019 ceph-mon[24574]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:50.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:50 smithi019 ceph-mon[29267]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:50.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:50 smithi172 ceph-mon[25457]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:51.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:51 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:36:51.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:51 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:36:51.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:51 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:36:52.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:52 smithi019 ceph-mon[24574]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:52.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:52 smithi019 ceph-mon[29267]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:52.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:52 smithi172 ceph-mon[25457]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:54.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:54 smithi172 ceph-mon[25457]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:54.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:54 smithi019 ceph-mon[24574]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:54.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:54 smithi019 ceph-mon[29267]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:56.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:56 smithi172 ceph-mon[25457]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:56.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:56 smithi019 ceph-mon[24574]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:56.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:56 smithi019 ceph-mon[29267]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:36:57.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:36:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:36:57] "GET /metrics HTTP/1.1" 200 37890 "" "Prometheus/2.33.4" 2024-02-07T12:36:58.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:58 smithi172 ceph-mon[25457]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:58.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:36:58 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:36:58.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:58 smithi019 ceph-mon[24574]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:58.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:36:58 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:36:58.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:58 smithi019 ceph-mon[29267]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:36:58.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:36:58 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:37:00.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:00 smithi172 ceph-mon[25457]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:00.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:00 smithi019 ceph-mon[24574]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:00.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:00 smithi019 ceph-mon[29267]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:02.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:02 smithi172 ceph-mon[25457]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:02.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:02 smithi019 ceph-mon[24574]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:02.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:02 smithi019 ceph-mon[29267]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:04.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:04 smithi172 ceph-mon[25457]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:37:04.396 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:04 smithi019 ceph-mon[24574]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:37:04.397 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:04 smithi019 ceph-mon[29267]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:37:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:06 smithi172 ceph-mon[25457]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:06 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:37:06.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:06 smithi019 ceph-mon[24574]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:06.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:06 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:37:06.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:06 smithi019 ceph-mon[29267]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:06.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:06 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:37:07.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:37:07] "GET /metrics HTTP/1.1" 200 37895 "" "Prometheus/2.33.4" 2024-02-07T12:37:08.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:08 smithi019 ceph-mon[24574]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:37:08.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:08 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:37:08.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:08 smithi019 ceph-mon[29267]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:37:08.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:08 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:37:08.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:08 smithi172 ceph-mon[25457]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:37:08.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:08 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:37:10.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:10 smithi019 ceph-mon[24574]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:10.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:10 smithi019 ceph-mon[29267]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:10.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:10 smithi172 ceph-mon[25457]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:12.421 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:12 smithi172 ceph-mon[25457]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:12.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:12 smithi019 ceph-mon[24574]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:12.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:12 smithi019 ceph-mon[29267]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:14.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:14 smithi019 ceph-mon[24574]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:37:14.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:14 smithi019 ceph-mon[29267]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:37:14.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:14 smithi172 ceph-mon[25457]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:37:16.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:16 smithi019 ceph-mon[24574]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:16.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:16 smithi019 ceph-mon[29267]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:16.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:16 smithi172 ceph-mon[25457]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:17.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:37:17] "GET /metrics HTTP/1.1" 200 37895 "" "Prometheus/2.33.4" 2024-02-07T12:37:18.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:18 smithi019 ceph-mon[24574]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:37:18.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:18 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:37:18.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:18 smithi019 ceph-mon[29267]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:37:18.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:18 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:37:18.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:18 smithi172 ceph-mon[25457]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:37:18.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:18 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:37:20.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:20 smithi019 ceph-mon[24574]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:20.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:20 smithi019 ceph-mon[29267]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:20.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:20 smithi172 ceph-mon[25457]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:21.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:21 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:37:21.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:21 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:37:21.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:21 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:37:22.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:22 smithi019 ceph-mon[24574]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:22.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:22 smithi019 ceph-mon[29267]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:22.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:22 smithi172 ceph-mon[25457]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:24.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:24 smithi019 ceph-mon[24574]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:37:24.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:24 smithi019 ceph-mon[29267]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:37:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:24 smithi172 ceph-mon[25457]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:37:26.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:26 smithi019 ceph-mon[24574]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:26.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:26 smithi019 ceph-mon[29267]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:26.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:26 smithi172 ceph-mon[25457]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:27.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:37:27] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.33.4" 2024-02-07T12:37:28.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:28 smithi019 ceph-mon[24574]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:37:28.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:28 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:37:28.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:28 smithi019 ceph-mon[29267]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:37:28.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:28 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:37:28.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:28 smithi172 ceph-mon[25457]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:37:28.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:28 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:37:30.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:30 smithi019 ceph-mon[24574]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:30.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:30 smithi019 ceph-mon[29267]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:30.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:30 smithi172 ceph-mon[25457]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:32.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:32 smithi019 ceph-mon[29267]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:32.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:32 smithi019 ceph-mon[24574]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:32.474 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch ps' 2024-02-07T12:37:32.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:32 smithi172 ceph-mon[25457]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:34.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:34 smithi019 ceph-mon[24574]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:37:34.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:34 smithi019 ceph-mon[29267]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:37:34.609 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:37:34.609 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (5m) 106s ago 13m 24.1M - 0.23.0 ba2b418f427c dfaf78b29ab5 2024-02-07T12:37:34.609 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (5m) 2m ago 13m 48.0M - 8.3.5 dad864ee21e9 e753e96fa4f0 2024-02-07T12:37:34.609 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 running (5m) 106s ago 12m 53.5M - 3.5 e1d6a67b021e 8072de108b37 2024-02-07T12:37:34.609 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283 running (8m) 2m ago 18m 507M - 19.0.0-1252-g44458cce 79957b372307 97644267e529 2024-02-07T12:37:34.609 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (2m) 106s ago 19m 449M - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:37:34.609 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (19m) 106s ago 19m 61.8M 2048M 17.2.0 e1d6a67b021e f3a8a050cf1d 2024-02-07T12:37:34.609 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (18m) 2m ago 18m 46.4M 2048M 17.2.0 e1d6a67b021e 7d62c7c4d075 2024-02-07T12:37:34.609 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (18m) 106s ago 18m 44.7M 2048M 17.2.0 e1d6a67b021e b9825d0db32d 2024-02-07T12:37:34.609 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (6m) 106s ago 14m 19.0M - 1.3.1 1dbe0e931976 9868df14bcc9 2024-02-07T12:37:34.609 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (5m) 2m ago 14m 17.7M - 1.3.1 1dbe0e931976 e066f1871320 2024-02-07T12:37:34.609 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (17m) 106s ago 17m 52.5M 2993M 17.2.0 e1d6a67b021e 87f2560a9b15 2024-02-07T12:37:34.610 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (17m) 106s ago 17m 55.2M 2993M 17.2.0 e1d6a67b021e 0f9a190f0e0d 2024-02-07T12:37:34.610 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (17m) 106s ago 17m 52.0M 2993M 17.2.0 e1d6a67b021e 7db94f25a1aa 2024-02-07T12:37:34.610 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (16m) 106s ago 16m 52.2M 2993M 17.2.0 e1d6a67b021e 0b6b674188ed 2024-02-07T12:37:34.610 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (16m) 2m ago 16m 52.9M 3248M 17.2.0 e1d6a67b021e df72cfd69b3b 2024-02-07T12:37:34.610 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (15m) 2m ago 15m 51.1M 3248M 17.2.0 e1d6a67b021e 3bdb22c07537 2024-02-07T12:37:34.610 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (15m) 2m ago 15m 49.3M 3248M 17.2.0 e1d6a67b021e 7859b651e0d9 2024-02-07T12:37:34.610 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (15m) 2m ago 15m 53.9M 3248M 17.2.0 e1d6a67b021e b301e7c75d73 2024-02-07T12:37:34.610 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (5m) 2m ago 14m 66.4M - 2.33.4 514e6a882f6e 929fbf32fae2 2024-02-07T12:37:34.610 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (13m) 106s ago 13m 75.1M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:37:34.610 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (12m) 2m ago 12m 73.6M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:37:34.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:34 smithi172 ceph-mon[25457]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:37:35.364 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph versions' 2024-02-07T12:37:36.569 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:36 smithi019 ceph-mon[29267]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:36.569 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:36 smithi019 ceph-mon[29267]: from='client.15171 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:37:36.569 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:36 smithi019 ceph-mon[29267]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:37:36.569 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:36 smithi019 ceph-mon[24574]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:36.569 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:36 smithi019 ceph-mon[24574]: from='client.15171 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:37:36.570 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:36 smithi019 ceph-mon[24574]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:37:36.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:36 smithi172 ceph-mon[25457]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:36.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:36 smithi172 ceph-mon[25457]: from='client.15171 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:37:36.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:36 smithi172 ceph-mon[25457]: from='mgr.24773 172.21.15.172:0/1955380951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:37:37.349 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:37:37.349 INFO:teuthology.orchestra.run.smithi019.stdout: "mon": { 2024-02-07T12:37:37.349 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-02-07T12:37:37.349 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:37:37.349 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr": { 2024-02-07T12:37:37.349 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:37:37.349 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:37:37.350 INFO:teuthology.orchestra.run.smithi019.stdout: "osd": { 2024-02-07T12:37:37.350 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-02-07T12:37:37.350 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:37:37.350 INFO:teuthology.orchestra.run.smithi019.stdout: "mds": {}, 2024-02-07T12:37:37.350 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw": { 2024-02-07T12:37:37.350 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:37:37.350 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:37:37.350 INFO:teuthology.orchestra.run.smithi019.stdout: "overall": { 2024-02-07T12:37:37.350 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-02-07T12:37:37.350 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:37:37.350 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:37:37.350 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:37:37.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:37:37] "GET /metrics HTTP/1.1" 200 37892 "" "Prometheus/2.33.4" 2024-02-07T12:37:37.977 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph health detail' 2024-02-07T12:37:38.277 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:38 smithi019 ceph-mon[29267]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:37:38.278 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:38 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:37:38.278 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:38 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/875103431' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:37:38.278 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:38 smithi019 ceph-mon[24574]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:37:38.278 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:38 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:37:38.278 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:38 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/875103431' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:37:38.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:38 smithi172 ceph-mon[25457]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:37:38.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:38 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:37:38.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:38 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/875103431' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:37:39.946 INFO:teuthology.orchestra.run.smithi019.stdout:HEALTH_OK 2024-02-07T12:37:40.179 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:40 smithi172 ceph-mon[25457]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:40.281 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:40 smithi019 ceph-mon[24574]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:40.281 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:40 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/633474207' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:37:40.281 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:40 smithi019 ceph-mon[29267]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:40.281 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:40 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/633474207' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:37:40.513 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph -s' 2024-02-07T12:37:40.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:40 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/633474207' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:37:42.501 INFO:teuthology.orchestra.run.smithi019.stdout: cluster: 2024-02-07T12:37:42.501 INFO:teuthology.orchestra.run.smithi019.stdout: id: ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:37:42.501 INFO:teuthology.orchestra.run.smithi019.stdout: health: HEALTH_OK 2024-02-07T12:37:42.502 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:37:42.502 INFO:teuthology.orchestra.run.smithi019.stdout: services: 2024-02-07T12:37:42.502 INFO:teuthology.orchestra.run.smithi019.stdout: mon: 3 daemons, quorum a,c,b (age 18m) 2024-02-07T12:37:42.502 INFO:teuthology.orchestra.run.smithi019.stdout: mgr: x(active, since 6m), standbys: y 2024-02-07T12:37:42.502 INFO:teuthology.orchestra.run.smithi019.stdout: osd: 8 osds: 8 up (since 15m), 8 in (since 15m) 2024-02-07T12:37:42.502 INFO:teuthology.orchestra.run.smithi019.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-02-07T12:37:42.502 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:37:42.502 INFO:teuthology.orchestra.run.smithi019.stdout: data: 2024-02-07T12:37:42.502 INFO:teuthology.orchestra.run.smithi019.stdout: pools: 6 pools, 161 pgs 2024-02-07T12:37:42.502 INFO:teuthology.orchestra.run.smithi019.stdout: objects: 209 objects, 585 KiB 2024-02-07T12:37:42.502 INFO:teuthology.orchestra.run.smithi019.stdout: usage: 94 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:37:42.502 INFO:teuthology.orchestra.run.smithi019.stdout: pgs: 161 active+clean 2024-02-07T12:37:42.503 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:37:42.503 INFO:teuthology.orchestra.run.smithi019.stdout: io: 2024-02-07T12:37:42.503 INFO:teuthology.orchestra.run.smithi019.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-02-07T12:37:42.503 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:37:42.503 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:42 smithi019 ceph-mon[24574]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:42.503 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:42 smithi019 ceph-mon[29267]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:42.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:42 smithi172 ceph-mon[25457]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:37:43.070 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph mgr fail' 2024-02-07T12:37:43.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:43 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2803261134' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-02-07T12:37:43.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:43 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2803261134' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-02-07T12:37:43.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:43 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2803261134' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-02-07T12:37:44.325 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:44 smithi019 ceph-mon[29267]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:37:44.325 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:44 smithi019 ceph-mon[24574]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:37:44.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:44 smithi172 ceph-mon[25457]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:37:45.456 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:45 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3636878027' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-02-07T12:37:45.456 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:45 smithi019 ceph-mon[24574]: osdmap e102: 8 total, 8 up, 8 in 2024-02-07T12:37:45.456 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:45 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3636878027' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-02-07T12:37:45.456 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:45 smithi019 ceph-mon[29267]: osdmap e102: 8 total, 8 up, 8 in 2024-02-07T12:37:45.457 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:45 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: [07/Feb/2024:12:37:45] ENGINE Bus STOPPING 2024-02-07T12:37:45.457 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:45 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: [07/Feb/2024:12:37:45] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-02-07T12:37:45.457 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:45 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: [07/Feb/2024:12:37:45] ENGINE Bus STOPPED 2024-02-07T12:37:45.457 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:45 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: [07/Feb/2024:12:37:45] ENGINE Bus STARTING 2024-02-07T12:37:45.457 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:45 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: [07/Feb/2024:12:37:45] ENGINE Serving on http://:::9283 2024-02-07T12:37:45.457 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:45 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: [07/Feb/2024:12:37:45] ENGINE Bus STARTED 2024-02-07T12:37:45.536 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:45 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3636878027' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-02-07T12:37:45.536 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:45 smithi172 ceph-mon[25457]: osdmap e102: 8 total, 8 up, 8 in 2024-02-07T12:37:45.536 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:45 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:37:45.195+0000 7f509efe7700 -1 mgr handle_mgr_map I was active but no longer am 2024-02-07T12:37:45.536 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:45 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ignoring --setuser ceph since I am not root 2024-02-07T12:37:45.536 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:45 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ignoring --setgroup ceph since I am not root 2024-02-07T12:37:45.536 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:45 smithi172 ceph-mgr[45648]: -- 172.21.15.172:0/3294133369 <== mon.2 v2:172.21.15.172:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x563f64e99520 con 0x563f64e6f000 2024-02-07T12:37:45.908 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'sleep 180' 2024-02-07T12:37:46.119 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:45 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:37:45.801+0000 7f060e09d200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-02-07T12:37:46.119 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:45 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:37:45.898+0000 7f060e09d200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-02-07T12:37:46.475 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3636878027' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-02-07T12:37:46.475 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-mon[25457]: mgrmap e29: y(active, starting, since 0.0930899s) 2024-02-07T12:37:46.475 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:37:46.475 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:37:46.475 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:37:46.475 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-02-07T12:37:46.475 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:37:46.475 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:37:46.476 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:37:46.476 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:37:46.476 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:37:46.476 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:37:46.476 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:37:46.476 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:37:46.476 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:37:46.476 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:37:46.476 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:37:46.476 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-mon[25457]: Manager daemon y is now available 2024-02-07T12:37:46.476 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:37:46.476 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:37:46.476 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:37:46.476 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:37:46.477 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:37:46.477 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:37:46.477 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:37:46.477 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3636878027' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-02-07T12:37:46.477 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[29267]: mgrmap e29: y(active, starting, since 0.0930899s) 2024-02-07T12:37:46.477 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:37:46.478 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:37:46.478 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:37:46.478 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-02-07T12:37:46.478 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:37:46.478 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:37:46.478 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:37:46.478 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:37:46.478 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:37:46.478 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:37:46.478 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:37:46.478 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:37:46.478 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:37:46.478 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:37:46.478 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:37:46.478 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[29267]: Manager daemon y is now available 2024-02-07T12:37:46.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:37:46.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:37:46.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:37:46.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:37:46.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:37:46.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:37:46.479 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:37:46.479 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3636878027' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-02-07T12:37:46.479 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[24574]: mgrmap e29: y(active, starting, since 0.0930899s) 2024-02-07T12:37:46.479 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:37:46.479 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:37:46.479 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:37:46.479 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-02-07T12:37:46.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:37:46.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:37:46.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:37:46.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:37:46.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:37:46.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:37:46.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:37:46.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:37:46.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:37:46.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:37:46.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:37:46.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[24574]: Manager daemon y is now available 2024-02-07T12:37:46.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:37:46.480 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:37:46.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:37:46.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:37:46.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:37:46.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:37:46.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:46 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:37:46.842 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:37:46.589+0000 7f060e09d200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-02-07T12:37:46.842 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:37:46.685+0000 7f060e09d200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-02-07T12:37:46.842 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:37:46.839+0000 7f060e09d200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-02-07T12:37:47.119 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:46 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:37:46.941+0000 7f060e09d200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-02-07T12:37:47.119 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:37:47.025+0000 7f060e09d200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-02-07T12:37:47.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:47 smithi019 ceph-mon[24574]: [07/Feb/2024:12:37:45] ENGINE Bus STARTING 2024-02-07T12:37:47.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:47 smithi019 ceph-mon[24574]: [07/Feb/2024:12:37:45] ENGINE Serving on http://172.21.15.19:8765 2024-02-07T12:37:47.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:47 smithi019 ceph-mon[24574]: [07/Feb/2024:12:37:45] ENGINE Serving on https://172.21.15.19:7150 2024-02-07T12:37:47.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:47 smithi019 ceph-mon[24574]: [07/Feb/2024:12:37:45] ENGINE Bus STARTED 2024-02-07T12:37:47.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:47 smithi019 ceph-mon[24574]: mgrmap e30: y(active, since 1.09862s) 2024-02-07T12:37:47.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:47 smithi019 ceph-mon[24574]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:37:47.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:47 smithi019 ceph-mon[29267]: [07/Feb/2024:12:37:45] ENGINE Bus STARTING 2024-02-07T12:37:47.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:47 smithi019 ceph-mon[29267]: [07/Feb/2024:12:37:45] ENGINE Serving on http://172.21.15.19:8765 2024-02-07T12:37:47.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:47 smithi019 ceph-mon[29267]: [07/Feb/2024:12:37:45] ENGINE Serving on https://172.21.15.19:7150 2024-02-07T12:37:47.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:47 smithi019 ceph-mon[29267]: [07/Feb/2024:12:37:45] ENGINE Bus STARTED 2024-02-07T12:37:47.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:47 smithi019 ceph-mon[29267]: mgrmap e30: y(active, since 1.09862s) 2024-02-07T12:37:47.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:47 smithi019 ceph-mon[29267]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:37:47.509 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:37:47.307+0000 7f060e09d200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-02-07T12:37:47.509 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:37:47.418+0000 7f060e09d200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-02-07T12:37:47.509 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:37:47.505+0000 7f060e09d200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-02-07T12:37:47.509 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:47 smithi172 ceph-mon[25457]: [07/Feb/2024:12:37:45] ENGINE Bus STARTING 2024-02-07T12:37:47.509 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:47 smithi172 ceph-mon[25457]: [07/Feb/2024:12:37:45] ENGINE Serving on http://172.21.15.19:8765 2024-02-07T12:37:47.509 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:47 smithi172 ceph-mon[25457]: [07/Feb/2024:12:37:45] ENGINE Serving on https://172.21.15.19:7150 2024-02-07T12:37:47.509 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:47 smithi172 ceph-mon[25457]: [07/Feb/2024:12:37:45] ENGINE Bus STARTED 2024-02-07T12:37:47.509 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:47 smithi172 ceph-mon[25457]: mgrmap e30: y(active, since 1.09862s) 2024-02-07T12:37:47.509 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:47 smithi172 ceph-mon[25457]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:37:47.811 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:37:47.609+0000 7f060e09d200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-02-07T12:37:47.811 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:37:47.808+0000 7f060e09d200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-02-07T12:37:48.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:48 smithi019 ceph-mon[29267]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:37:48.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:48 smithi019 ceph-mon[29267]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:37:48.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:48 smithi019 ceph-mon[29267]: mgrmap e31: y(active, since 2s) 2024-02-07T12:37:48.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:48 smithi019 ceph-mon[24574]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:37:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:48 smithi019 ceph-mon[24574]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:37:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:48 smithi019 ceph-mon[24574]: mgrmap e31: y(active, since 2s) 2024-02-07T12:37:48.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:48 smithi172 ceph-mon[25457]: from='client.24979 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:37:48.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:48 smithi172 ceph-mon[25457]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:37:48.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:48 smithi172 ceph-mon[25457]: mgrmap e31: y(active, since 2s) 2024-02-07T12:37:48.948 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:48 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:37:48.812+0000 7f060e09d200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-02-07T12:37:48.948 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:48 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:37:48.945+0000 7f060e09d200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-02-07T12:37:49.598 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:49 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:37:49.293+0000 7f060e09d200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-02-07T12:37:49.599 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:49 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:37:49.386+0000 7f060e09d200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-02-07T12:37:49.869 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:49 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:37:49.595+0000 7f060e09d200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:37:49.869 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:49 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:37:49.787+0000 7f060e09d200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-02-07T12:37:50.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:50 smithi019 ceph-mon[29267]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:37:50.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:50 smithi019 ceph-mon[29267]: mgrmap e32: y(active, since 4s) 2024-02-07T12:37:50.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:50 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:50.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:50 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:50.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:50 smithi019 ceph-mon[24574]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:37:50.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:50 smithi019 ceph-mon[24574]: mgrmap e32: y(active, since 4s) 2024-02-07T12:37:50.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:50 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:50.339 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:50 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:50.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:50 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:37:50.094+0000 7f060e09d200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:37:50.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:50 smithi172 ceph-mon[25457]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:37:50.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:50 smithi172 ceph-mon[25457]: mgrmap e32: y(active, since 4s) 2024-02-07T12:37:50.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:50 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:50.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:50 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:50.627 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:50 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:37:50.372+0000 7f060e09d200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-02-07T12:37:51.119 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:50 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:37:50.624+0000 7f060e09d200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-02-07T12:37:51.869 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:51 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:37:51.452+0000 7f060e09d200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-02-07T12:37:52.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:52 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:52.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:52 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:52.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:52 smithi019 ceph-mon[24574]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:37:52.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:52 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:52.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:52 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:52.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:52 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:52.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:52 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:52.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:52 smithi019 ceph-mon[29267]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:37:52.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:52 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:52.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:52 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:52.351 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:52 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:52.351 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:52 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:52.351 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:52 smithi172 ceph-mon[25457]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:37:52.352 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:52 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:52.352 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:52 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:52.352 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:52 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:37:52.029+0000 7f060e09d200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-02-07T12:37:52.352 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:52 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:37:52.127+0000 7f060e09d200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-02-07T12:37:52.352 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:52 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:37:52.348+0000 7f060e09d200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-02-07T12:37:52.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:52 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:37:52] ENGINE Bus STARTING 2024-02-07T12:37:52.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:52 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: CherryPy Checker: 2024-02-07T12:37:52.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:52 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: The Application mounted at '' has an empty config. 2024-02-07T12:37:52.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:52 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:37:52.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:52 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:37:52] ENGINE Serving on http://:::9283 2024-02-07T12:37:52.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:52 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:37:52] ENGINE Bus STARTED 2024-02-07T12:37:53.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:53 smithi019 ceph-mon[24574]: Standby manager daemon x started 2024-02-07T12:37:53.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:53 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-02-07T12:37:53.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:53 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:37:53.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:53 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-02-07T12:37:53.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:53 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:37:53.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:53 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:53.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:53 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:53.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:53 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:37:53.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:53 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:37:53.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:53 smithi019 ceph-mon[24574]: Updating smithi019:/etc/ceph/ceph.conf 2024-02-07T12:37:53.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:53 smithi019 ceph-mon[24574]: Updating smithi172:/etc/ceph/ceph.conf 2024-02-07T12:37:53.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:53 smithi019 ceph-mon[29267]: Standby manager daemon x started 2024-02-07T12:37:53.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:53 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-02-07T12:37:53.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:53 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:37:53.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:53 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-02-07T12:37:53.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:53 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:37:53.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:53 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:53.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:53 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:53.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:53 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:37:53.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:53 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:37:53.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:53 smithi019 ceph-mon[29267]: Updating smithi019:/etc/ceph/ceph.conf 2024-02-07T12:37:53.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:53 smithi019 ceph-mon[29267]: Updating smithi172:/etc/ceph/ceph.conf 2024-02-07T12:37:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:53 smithi172 ceph-mon[25457]: Standby manager daemon x started 2024-02-07T12:37:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:53 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-02-07T12:37:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:53 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:37:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:53 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-02-07T12:37:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:53 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:37:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:53 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:53 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:53 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:37:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:53 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:37:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:53 smithi172 ceph-mon[25457]: Updating smithi019:/etc/ceph/ceph.conf 2024-02-07T12:37:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:53 smithi172 ceph-mon[25457]: Updating smithi172:/etc/ceph/ceph.conf 2024-02-07T12:37:54.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[29267]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:37:54.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[29267]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:37:54.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[29267]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:37:54.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[29267]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:37:54.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[29267]: mgrmap e33: y(active, since 7s), standbys: x 2024-02-07T12:37:54.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-02-07T12:37:54.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[29267]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-02-07T12:37:54.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[29267]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:37:54.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[29267]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:37:54.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:54.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:54.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:54.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:54.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:54.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:37:54.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:37:54.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:37:54.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[24574]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:37:54.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[24574]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:37:54.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[24574]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:37:54.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[24574]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:37:54.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[24574]: mgrmap e33: y(active, since 7s), standbys: x 2024-02-07T12:37:54.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-02-07T12:37:54.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[24574]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-02-07T12:37:54.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[24574]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:37:54.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[24574]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:37:54.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:54.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:54.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:54.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:54.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:54.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:37:54.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:37:54.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:54 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:37:54.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:54 smithi172 ceph-mon[25457]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:37:54.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:54 smithi172 ceph-mon[25457]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:37:54.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:54 smithi172 ceph-mon[25457]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:37:54.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:54 smithi172 ceph-mon[25457]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:37:54.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:54 smithi172 ceph-mon[25457]: mgrmap e33: y(active, since 7s), standbys: x 2024-02-07T12:37:54.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:54 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-02-07T12:37:54.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:54 smithi172 ceph-mon[25457]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-02-07T12:37:54.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:54 smithi172 ceph-mon[25457]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:37:54.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:54 smithi172 ceph-mon[25457]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:37:54.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:54 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:54.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:54 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:54.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:54 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:54.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:54 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:54.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:54 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:37:54.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:54 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:37:54.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:54 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:37:54.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:54 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:37:55.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:55 smithi019 ceph-mon[24574]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:37:55.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:55 smithi019 ceph-mon[24574]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:37:55.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:55 smithi019 ceph-mon[29267]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:37:55.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:55 smithi019 ceph-mon[29267]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:37:55.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:55 smithi172 ceph-mon[25457]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:37:55.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:55 smithi172 ceph-mon[25457]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:37:56.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:56 smithi172 ceph-mon[25457]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-02-07T12:37:56.459 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-mon[29267]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-02-07T12:37:56.459 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-mon[24574]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-02-07T12:37:56.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:37:56.594+0000 7fcce5b20700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:37:56.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:37:56.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:37:56.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:37:56.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:37:56.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:37:56.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:37:56.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:37:56.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:37:56.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:37:56.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:37:56.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:37:56.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:37:56.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:37:56.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:37:56.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:37:56.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:37:56.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:37:56.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:37:56.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:37:56.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:37:56.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:37:56.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:37:56.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:37:56.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:37:56.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:37:56.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:37:56.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:37:56.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:37:56.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:37:56.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:37:56.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:37:56.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:37:56.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:37:56.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:37:56.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:37:56.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:37:56.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:37:56.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:37:56.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:37:56.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:37:56.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:37:56.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:37:56.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:37:56.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:37:56.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:37:56.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:37:56.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:37:56.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:37:56.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:37:56.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:37:56.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:37:56.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:37:56.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:37:56.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:37:56.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:37:56.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:37:56.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:37:56.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:37:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:37:57.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:37:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:37:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:37:58.530 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:58 smithi172 ceph-mon[25457]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:37:58.530 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:37:58.530 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:37:58.530 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:37:58.530 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:37:58.530 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:37:58.530 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:37:58.530 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:37:58.530 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:37:58.531 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:37:58.531 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:37:58.531 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:37:58.531 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:37:58.531 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:37:58.531 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:37:58.531 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:37:58.531 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:37:58.531 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:37:58.531 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:37:58.531 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:37:58.531 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:37:58.531 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:37:58.531 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:37:58.532 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:37:58.532 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:37:58.532 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:37:58.532 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:37:58.532 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:37:58.532 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:37:58.532 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:37:58.532 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:37:58.532 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:37:58.532 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:37:58.532 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:37:58.532 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:37:58.532 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:37:58.532 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:37:58.532 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:37:58.533 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:37:58.533 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:37:58.533 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:37:58.533 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:37:58.533 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:37:58.533 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:37:58.533 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:37:58.533 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:37:58.533 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:37:58.533 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:37:58.533 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:37:58.533 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:37:58.533 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:37:58.533 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:37:58.533 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:37:58.534 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:37:58.534 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:37:58.534 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:37:58.534 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:37:58.534 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:37:58.534 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:37:58.534 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:58 smithi172 ceph-mon[25457]: Reconfiguring prometheus.a (dependencies changed)... 2024-02-07T12:37:58.534 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:58 smithi172 ceph-mon[25457]: Reconfiguring daemon prometheus.a on smithi172 2024-02-07T12:37:58.534 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:37:58 smithi172 ceph-mon[25457]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-02-07T12:37:58.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:58 smithi019 ceph-mon[29267]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:37:58.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:37:58.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:37:58.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:37:58.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:37:58.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:37:58.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:37:58.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:37:58.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:37:58.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:37:58.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:37:58.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:37:58.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:37:58.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:37:58.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:37:58.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:37:58.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:37:58.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:37:58.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:37:58.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:37:58.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:37:58.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:37:58.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:37:58.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:37:58.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:37:58.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:37:58.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:37:58.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:37:58.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:37:58.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:37:58.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:37:58.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:37:58.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:37:58.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:37:58.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:37:58.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:37:58.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:37:58.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:37:58.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:37:58.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:37:58.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:37:58.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:37:58.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:37:58.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:37:58.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:37:58.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:37:58.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:37:58.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:37:58.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:37:58.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:37:58.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:37:58.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:37:58.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:37:58.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:37:58.598 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:58 smithi019 ceph-mon[24574]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:37:58.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:37:58.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:37:58.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:37:58.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:37:58.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:37:58.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:37:58.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:37:58.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:37:58.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:37:58.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:37:58.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:37:58.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:37:58.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:37:58.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:37:58.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:37:58.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:37:58.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:37:58.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:37:58.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:37:58.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:37:58.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:37:58.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:37:58.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:37:58.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:37:58.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:37:58.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:37:58.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:37:58.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:37:58.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:37:58.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:37:58.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:37:58.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:37:58.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:37:58.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:37:58.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:37:58.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:37:58.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:37:58.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:37:58.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:37:58.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:37:58.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:37:58.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:37:58.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:37:58.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:37:58.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:37:58.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:37:58.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:37:58.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:37:58.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:37:58.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:37:58.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:37:58.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:37:58.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:37:58.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:37:58.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:37:58.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:37:58.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:37:58.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:37:58.602 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:58 smithi019 ceph-mon[29267]: Reconfiguring prometheus.a (dependencies changed)... 2024-02-07T12:37:58.602 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:58 smithi019 ceph-mon[29267]: Reconfiguring daemon prometheus.a on smithi172 2024-02-07T12:37:58.602 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:37:58 smithi019 ceph-mon[29267]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-02-07T12:37:58.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:37:58.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:37:58.603 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:37:58.603 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:37:58.603 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:37:58.603 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:58 smithi019 ceph-mon[24574]: Reconfiguring prometheus.a (dependencies changed)... 2024-02-07T12:37:58.603 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:58 smithi019 ceph-mon[24574]: Reconfiguring daemon prometheus.a on smithi172 2024-02-07T12:37:58.603 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:37:58 smithi019 ceph-mon[24574]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-02-07T12:37:58.857 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:37:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:37:58.682Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.172:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.172:8765: connect: connection refused" 2024-02-07T12:37:58.857 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:37:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:37:58.683Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.172:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.172:8765: connect: connection refused" 2024-02-07T12:37:58.857 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:37:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:37:58.685Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.172:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.172:8765: connect: connection refused" 2024-02-07T12:37:58.857 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:37:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:37:58.685Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.172:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.172:8765: connect: connection refused" 2024-02-07T12:38:00.263 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:00 smithi172 systemd[1]: Stopping Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:38:00.264 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:00 smithi172 bash[54243]: Error: no container with name or ID "ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus.a" found: no such container 2024-02-07T12:38:00.587 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:00 smithi172 ceph-mon[25457]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-02-07T12:38:00.588 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:38:00.285Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-02-07T12:38:00.588 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:38:00.285Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-02-07T12:38:00.588 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:38:00.285Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-02-07T12:38:00.588 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:38:00.285Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-02-07T12:38:00.588 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:38:00.285Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-02-07T12:38:00.588 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:38:00.285Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-02-07T12:38:00.588 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:38:00.285Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-02-07T12:38:00.588 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:38:00.285Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-02-07T12:38:00.588 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:38:00.286Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-02-07T12:38:00.588 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:38:00.286Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-02-07T12:38:00.588 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:38:00.286Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-02-07T12:38:00.589 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[48906]: ts=2024-02-07T12:38:00.286Z caller=main.go:1066 level=info msg="See you next time!" 2024-02-07T12:38:00.589 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:00 smithi172 podman[54256]: 2024-02-07 12:38:00.295879928 +0000 UTC m=+0.114288257 container died 929fbf32fae22e1d02ece26d158a1699b29c95b22fbfbb22f799213140313054 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:38:00.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:00 smithi019 ceph-mon[29267]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-02-07T12:38:00.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:00 smithi019 ceph-mon[24574]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-02-07T12:38:00.872 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:00 smithi172 podman[54256]: 2024-02-07 12:38:00.685153427 +0000 UTC m=+0.503561762 container cleanup 929fbf32fae22e1d02ece26d158a1699b29c95b22fbfbb22f799213140313054 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:38:00.872 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:00 smithi172 bash[54256]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a 2024-02-07T12:38:00.872 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:00 smithi172 bash[54285]: Error: no container with name or ID "ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus.a" found: no such container 2024-02-07T12:38:01.268 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:00 smithi172 podman[54270]: 2024-02-07 12:38:00.911466726 +0000 UTC m=+0.618179178 container remove 929fbf32fae22e1d02ece26d158a1699b29c95b22fbfbb22f799213140313054 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:38:01.268 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:00 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@prometheus.a.service: Deactivated successfully. 2024-02-07T12:38:01.268 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:00 smithi172 systemd[1]: Stopped Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:38:01.268 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:00 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@prometheus.a.service: Consumed 1.841s CPU time. 2024-02-07T12:38:01.268 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:00 smithi172 systemd[1]: Starting Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:38:01.268 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 podman[54355]: 2024-02-07 12:38:01.046164508 +0000 UTC m=+0.019351902 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-02-07T12:38:01.268 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 podman[54355]: 2024-02-07 12:38:01.145725123 +0000 UTC m=+0.118912585 container create 7371d5c3aec06aed9ba002c759b308247210d2c2e63f7df565282288abbb1fdb (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:38:01.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:01 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:38:01.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:01 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:38:01.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:01 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:38:01.619 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 podman[54355]: 2024-02-07 12:38:01.381508239 +0000 UTC m=+0.354695647 container init 7371d5c3aec06aed9ba002c759b308247210d2c2e63f7df565282288abbb1fdb (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:38:01.619 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 podman[54355]: 2024-02-07 12:38:01.384067337 +0000 UTC m=+0.357254730 container start 7371d5c3aec06aed9ba002c759b308247210d2c2e63f7df565282288abbb1fdb (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:38:01.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:38:01.406Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-02-07T12:38:01.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:38:01.406Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-02-07T12:38:01.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:38:01.406Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-02-07T12:38:01.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:38:01.406Z caller=main.go:518 level=info host_details="(Linux 5.14.0-418.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Mon Feb 5 23:17:48 UTC 2024 x86_64 smithi172 (none))" 2024-02-07T12:38:01.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:38:01.406Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-02-07T12:38:01.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:38:01.406Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-02-07T12:38:01.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:38:01.408Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-02-07T12:38:01.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:38:01.408Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-02-07T12:38:01.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:38:01.409Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-02-07T12:38:01.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:38:01.412Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-02-07T12:38:01.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:38:01.412Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.606µs 2024-02-07T12:38:01.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:38:01.412Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-02-07T12:38:01.621 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 bash[54355]: 7371d5c3aec06aed9ba002c759b308247210d2c2e63f7df565282288abbb1fdb 2024-02-07T12:38:01.621 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 systemd[1]: Started Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:38:01.922 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:38:01.648Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-02-07T12:38:01.922 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:38:01.872Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-02-07T12:38:01.922 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:38:01.920Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-02-07T12:38:02.328 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:38:01.921Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-02-07T12:38:02.328 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:38:01.921Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=30.909µs wal_replay_duration=509.004382ms total_replay_duration=509.048179ms 2024-02-07T12:38:02.328 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:38:01.926Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-02-07T12:38:02.328 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:38:01.926Z caller=main.go:947 level=info msg="TSDB started" 2024-02-07T12:38:02.329 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:38:01.926Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-02-07T12:38:02.329 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:38:01.948Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=22.51561ms db_storage=855ns remote_storage=1.673µs web_handler=339ns query_engine=697ns scrape=232.907µs scrape_sd=114.857µs notify=36.638µs notify_sd=20.56µs rules=21.769776ms 2024-02-07T12:38:02.329 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:38:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:38:01.948Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-02-07T12:38:02.603 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:02 smithi172 ceph-mon[25457]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-02-07T12:38:02.603 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:02 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:38:02.603 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:02 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:38:02.604 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:02 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:38:02.604 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:02 smithi172 ceph-mon[25457]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:38:02.604 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:02 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:38:02.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:02 smithi019 ceph-mon[29267]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-02-07T12:38:02.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:02 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:38:02.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:02 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:38:02.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:02 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:38:02.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:02 smithi019 ceph-mon[29267]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:38:02.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:02 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:38:02.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:02 smithi019 ceph-mon[24574]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-02-07T12:38:02.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:02 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:38:02.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:02 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:38:02.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:02 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:38:02.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:02 smithi019 ceph-mon[24574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:38:02.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:02 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:38:04.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:04 smithi172 ceph-mon[25457]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-02-07T12:38:04.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:04 smithi019 ceph-mon[29267]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-02-07T12:38:04.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:04 smithi019 ceph-mon[24574]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-02-07T12:38:06.546 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:06 smithi019 ceph-mon[24574]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-02-07T12:38:06.546 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:06 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:38:06.546 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:06 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:38:06.547 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:06 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:38:06.547 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:06 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:38:06.547 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:06 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:38:06.547 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:06 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:38:06.547 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:06 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:38:06.547 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:06 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:38:06.547 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:06 smithi019 ceph-mon[29267]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-02-07T12:38:06.547 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:06 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:38:06.547 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:06 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:38:06.547 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:06 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:38:06.547 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:06 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:38:06.547 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:06 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:38:06.548 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:06 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:38:06.548 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:06 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:38:06.548 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:06 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:38:06.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:06 smithi172 ceph-mon[25457]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-02-07T12:38:06.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:06 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:38:06.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:06 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:38:06.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:06 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:38:06.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:06 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:38:06.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:06 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:38:06.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:06 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:38:06.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:06 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:38:06.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:06 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:38:07.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:07 smithi019 ceph-mon[29267]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:38:07.588 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:07 smithi019 ceph-mon[29267]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:38:07.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:07 smithi019 ceph-mon[24574]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:38:07.588 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:07 smithi019 ceph-mon[24574]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:38:07.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:07 smithi172 ceph-mon[25457]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:38:07.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:07 smithi172 ceph-mon[25457]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:38:08.531 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:08 smithi019 ceph-mon[24574]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-02-07T12:38:08.531 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:08 smithi019 ceph-mon[29267]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-02-07T12:38:08.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:08 smithi172 ceph-mon[25457]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-02-07T12:38:10.167 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:38:10.047+0000 7fcce5b20700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi-foo-smithi019-svjimj 2024-02-07T12:38:10.167 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: /usr/bin/podman: stderr Error: no such container ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi-foo-smithi019-svjimj 2024-02-07T12:38:10.168 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi.foo.smithi019.svjimj 2024-02-07T12:38:10.168 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: /usr/bin/podman: stderr Error: no such container ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi.foo.smithi019.svjimj 2024-02-07T12:38:10.168 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:38:10.168 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:38:10.168 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:38:10.168 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:38:10.168 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:38:10.168 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:38:10.168 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:38:10.168 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:38:10.168 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:38:10.168 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:38:10.169 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:38:10.169 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:38:10.169 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:38:10.169 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:38:10.169 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:38:10.169 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:38:10.170 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:38:10.170 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:38:10.170 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:38:10.170 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:38:10.170 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:38:10.170 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:38:10.170 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:38:10.170 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:38:10.170 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:38:10.170 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:38:10.170 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:38:10.170 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:38:10.170 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:38:10.170 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:38:10.171 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:38:10.171 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:38:10.171 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:38:10.171 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:38:10.171 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:38:10.171 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:38:10.171 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:38:10.171 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:38:10.171 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi-foo-smithi019-svjimj 2024-02-07T12:38:10.171 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: /usr/bin/podman: stderr Error: no such container ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi-foo-smithi019-svjimj 2024-02-07T12:38:10.171 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi.foo.smithi019.svjimj 2024-02-07T12:38:10.171 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: /usr/bin/podman: stderr Error: no such container ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi.foo.smithi019.svjimj 2024-02-07T12:38:10.171 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:38:10.171 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:38:10.171 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:38:10.171 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:38:10.172 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:38:10.172 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:38:10.172 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:38:10.172 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:38:10.172 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:38:10.172 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:38:10.172 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:38:10.172 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:38:10.172 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:38:10.172 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:38:10.172 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:38:10.172 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:38:10.172 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:38:10.172 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:38:10.173 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:38:10.173 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:38:10.173 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:38:10.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-mon[29267]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:38:10.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3246800867' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-02-07T12:38:10.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-mon[24574]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:38:10.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:10 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3246800867' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-02-07T12:38:10.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:10 smithi172 ceph-mon[25457]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:38:10.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:10 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3246800867' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-02-07T12:38:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:11 smithi019 ceph-mon[24574]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi-foo-smithi019-svjimj 2024-02-07T12:38:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: /usr/bin/podman: stderr Error: no such container ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi-foo-smithi019-svjimj 2024-02-07T12:38:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi.foo.smithi019.svjimj 2024-02-07T12:38:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: /usr/bin/podman: stderr Error: no such container ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi.foo.smithi019.svjimj 2024-02-07T12:38:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:38:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:38:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:38:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:38:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:38:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:38:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:38:11.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:38:11.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:38:11.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:38:11.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:38:11.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:38:11.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:38:11.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:38:11.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:38:11.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:38:11.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:38:11.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:38:11.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:38:11.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:38:11.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:38:11.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:38:11.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:38:11.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:38:11.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:38:11.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:38:11.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:38:11.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:38:11.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:38:11.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:38:11.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:38:11.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:38:11.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:38:11.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:38:11.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:38:11.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:38:11.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:38:11.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:38:11.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi-foo-smithi019-svjimj 2024-02-07T12:38:11.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: /usr/bin/podman: stderr Error: no such container ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi-foo-smithi019-svjimj 2024-02-07T12:38:11.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi.foo.smithi019.svjimj 2024-02-07T12:38:11.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: /usr/bin/podman: stderr Error: no such container ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi.foo.smithi019.svjimj 2024-02-07T12:38:11.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:38:11.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:38:11.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:38:11.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:38:11.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:38:11.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:38:11.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:38:11.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:38:11.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:38:11.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:38:11.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:38:11.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:38:11.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:38:11.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:38:11.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:38:11.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:38:11.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:38:11.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:38:11.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:38:11.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:38:11.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:38:11.599 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:11 smithi019 ceph-mon[29267]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi-foo-smithi019-svjimj 2024-02-07T12:38:11.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: /usr/bin/podman: stderr Error: no such container ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi-foo-smithi019-svjimj 2024-02-07T12:38:11.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi.foo.smithi019.svjimj 2024-02-07T12:38:11.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: /usr/bin/podman: stderr Error: no such container ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi.foo.smithi019.svjimj 2024-02-07T12:38:11.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:38:11.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:38:11.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:38:11.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:38:11.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:38:11.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:38:11.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:38:11.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:38:11.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:38:11.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:38:11.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:38:11.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:38:11.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:38:11.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:38:11.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:38:11.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:38:11.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:38:11.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:38:11.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:38:11.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:38:11.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:38:11.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:38:11.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:38:11.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:38:11.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:38:11.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:38:11.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:38:11.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:38:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:38:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:38:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:38:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:38:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:38:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:38:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:38:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:38:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:38:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:38:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi-foo-smithi019-svjimj 2024-02-07T12:38:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: /usr/bin/podman: stderr Error: no such container ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi-foo-smithi019-svjimj 2024-02-07T12:38:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi.foo.smithi019.svjimj 2024-02-07T12:38:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: /usr/bin/podman: stderr Error: no such container ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi.foo.smithi019.svjimj 2024-02-07T12:38:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:38:11.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:38:11.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:38:11.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:38:11.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:38:11.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:38:11.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:38:11.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:38:11.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:38:11.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:38:11.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:38:11.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:38:11.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:38:11.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:38:11.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:38:11.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:38:11.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:38:11.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:38:11.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:38:11.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:38:11.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:38:11.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:11 smithi172 ceph-mon[25457]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi-foo-smithi019-svjimj 2024-02-07T12:38:11.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: /usr/bin/podman: stderr Error: no such container ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi-foo-smithi019-svjimj 2024-02-07T12:38:11.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi.foo.smithi019.svjimj 2024-02-07T12:38:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: /usr/bin/podman: stderr Error: no such container ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi.foo.smithi019.svjimj 2024-02-07T12:38:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:38:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:38:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:38:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:38:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:38:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:38:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:38:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:38:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:38:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:38:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:38:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:38:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:38:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:38:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:38:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:38:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:38:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:38:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:38:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:38:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:38:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:38:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:38:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:38:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:38:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:38:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:38:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:38:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:38:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:38:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:38:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:38:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:38:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:38:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:38:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:38:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:38:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:38:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi-foo-smithi019-svjimj 2024-02-07T12:38:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: /usr/bin/podman: stderr Error: no such container ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi-foo-smithi019-svjimj 2024-02-07T12:38:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi.foo.smithi019.svjimj 2024-02-07T12:38:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: /usr/bin/podman: stderr Error: no such container ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-iscsi.foo.smithi019.svjimj 2024-02-07T12:38:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:38:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:38:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:38:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:38:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:38:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:38:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:38:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:38:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:38:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:38:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:38:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:38:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:38:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:38:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:38:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:38:11.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:38:11.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:38:11.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:38:11.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:38:11.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:38:12.542 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:12 smithi172 ceph-mon[25457]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:38:12.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:12 smithi019 ceph-mon[29267]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:38:12.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:12 smithi019 ceph-mon[24574]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:38:14.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:14 smithi019 ceph-mon[29267]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:14.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:14 smithi019 ceph-mon[24574]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:14.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:38:13] "GET /metrics HTTP/1.1" 200 37901 "" "Prometheus/2.33.4" 2024-02-07T12:38:14.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:14 smithi172 ceph-mon[25457]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:16.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:16 smithi019 ceph-mon[24574]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:16.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:16 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:38:16.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:16 smithi019 ceph-mon[29267]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:16.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:16 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:38:16.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:16 smithi172 ceph-mon[25457]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:16.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:16 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:38:18.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:18 smithi019 ceph-mon[24574]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-02-07T12:38:18.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:18 smithi019 ceph-mon[29267]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-02-07T12:38:18.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:18 smithi172 ceph-mon[25457]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-02-07T12:38:20.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:20 smithi019 ceph-mon[29267]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-02-07T12:38:20.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:20 smithi019 ceph-mon[24574]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-02-07T12:38:20.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:20 smithi172 ceph-mon[25457]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-02-07T12:38:21.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:21 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:38:21.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:21 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:38:21.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:21 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:38:22.571 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:22 smithi172 ceph-mon[25457]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-02-07T12:38:22.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:22 smithi019 ceph-mon[29267]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-02-07T12:38:22.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:22 smithi019 ceph-mon[24574]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-02-07T12:38:24.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:24 smithi019 ceph-mon[24574]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-02-07T12:38:24.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:24 smithi019 ceph-mon[29267]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-02-07T12:38:24.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:38:23] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-02-07T12:38:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:24 smithi172 ceph-mon[25457]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-02-07T12:38:26.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:26 smithi019 ceph-mon[24574]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:38:26.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:26 smithi019 ceph-mon[29267]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:38:26.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:26 smithi172 ceph-mon[25457]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:38:28.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:28 smithi019 ceph-mon[24574]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:38:28.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:28 smithi019 ceph-mon[29267]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:38:28.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:28 smithi172 ceph-mon[25457]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:38:30.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:30 smithi019 ceph-mon[24574]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:38:30.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:30 smithi019 ceph-mon[29267]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:38:30.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:30 smithi172 ceph-mon[25457]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:38:31.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:31 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:38:31.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:31 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:38:31.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:31 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:38:31.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:31 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:38:31.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:31 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:38:31.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:31 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:38:32.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:32 smithi019 ceph-mon[24574]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:38:32.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:32 smithi019 ceph-mon[29267]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:38:32.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:32 smithi172 ceph-mon[25457]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:38:34.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:34 smithi019 ceph-mon[29267]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:38:34.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:34 smithi019 ceph-mon[24574]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:38:34.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:38:33] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.33.4" 2024-02-07T12:38:34.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:34 smithi172 ceph-mon[25457]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:38:36.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:36 smithi019 ceph-mon[24574]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:38:36.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:36 smithi019 ceph-mon[29267]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:38:36.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:36 smithi172 ceph-mon[25457]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:38:38.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:38 smithi019 ceph-mon[29267]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:38:38.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:38 smithi019 ceph-mon[24574]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:38:38.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:38 smithi172 ceph-mon[25457]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:38:40.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:40 smithi019 ceph-mon[29267]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:38:40.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:40 smithi019 ceph-mon[24574]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:38:40.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:40 smithi172 ceph-mon[25457]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:38:41.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:41 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:38:41.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:41 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:38:41.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:41 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:38:42.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:42 smithi019 ceph-mon[24574]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:38:42.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:42 smithi019 ceph-mon[29267]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:38:42.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:42 smithi172 ceph-mon[25457]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:38:44.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:44 smithi019 ceph-mon[24574]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:38:44.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:44 smithi019 ceph-mon[29267]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:38:44.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:38:43] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.33.4" 2024-02-07T12:38:44.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:44 smithi172 ceph-mon[25457]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:38:46.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:46 smithi019 ceph-mon[24574]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:38:46.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:46 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:38:46.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:46 smithi019 ceph-mon[29267]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:38:46.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:46 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:38:46.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:46 smithi172 ceph-mon[25457]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:38:46.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:46 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:38:48.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:48 smithi019 ceph-mon[24574]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:38:48.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:48 smithi019 ceph-mon[29267]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:38:48.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:48 smithi172 ceph-mon[25457]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:38:50.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:50 smithi019 ceph-mon[24574]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:38:50.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:50 smithi019 ceph-mon[29267]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:38:50.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:50 smithi172 ceph-mon[25457]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:38:51.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:51 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:38:51.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:51 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:38:51.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:51 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:38:52.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:52 smithi019 ceph-mon[24574]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:38:52.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:52 smithi019 ceph-mon[29267]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:38:52.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:52 smithi172 ceph-mon[25457]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:38:54.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:54 smithi019 ceph-mon[24574]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:38:54.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:38:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:38:53] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-02-07T12:38:54.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:54 smithi019 ceph-mon[29267]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:38:54.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:54 smithi172 ceph-mon[25457]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:38:56.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:56 smithi019 ceph-mon[29267]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:38:56.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:56 smithi019 ceph-mon[24574]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:38:56.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:56 smithi172 ceph-mon[25457]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:38:58.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:38:58 smithi019 ceph-mon[29267]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:38:58.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:38:58 smithi019 ceph-mon[24574]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:38:58.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:38:58 smithi172 ceph-mon[25457]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:00.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:00 smithi019 ceph-mon[29267]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:00.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:00 smithi019 ceph-mon[24574]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:00.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:00 smithi172 ceph-mon[25457]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:01.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:01 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:39:01.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:01 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:39:01.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:01 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:39:01.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:01 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:39:01.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:01 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:39:01.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:01 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:39:02.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:02 smithi019 ceph-mon[29267]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:02.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:02 smithi019 ceph-mon[24574]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:02.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:02 smithi172 ceph-mon[25457]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:04.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:04 smithi019 ceph-mon[24574]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:04.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:39:03] "GET /metrics HTTP/1.1" 200 37895 "" "Prometheus/2.33.4" 2024-02-07T12:39:04.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:04 smithi019 ceph-mon[29267]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:04.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:04 smithi172 ceph-mon[25457]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:06.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:06 smithi019 ceph-mon[24574]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:06.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:06 smithi019 ceph-mon[29267]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:06.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:06 smithi172 ceph-mon[25457]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:08.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:08 smithi019 ceph-mon[24574]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:08.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:08 smithi019 ceph-mon[29267]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:08.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:08 smithi172 ceph-mon[25457]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:10.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:10 smithi019 ceph-mon[24574]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:10.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:10 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:39:10.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:10 smithi019 ceph-mon[29267]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:10.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:10 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:39:10.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:10 smithi172 ceph-mon[25457]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:10.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:10 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:39:11.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:11 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:39:11.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:11 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:39:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:11 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:39:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:11 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:39:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:11 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:39:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:11 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:39:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:11 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:39:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:11 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:39:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:11 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:39:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:11 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:39:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:11 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:39:11.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:11 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:39:11.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:11 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:39:11.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:11 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:39:11.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:11 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:39:11.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:11 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:39:11.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:11 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:39:11.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:11 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:39:11.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:11 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:39:11.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:11 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:39:11.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:11 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:39:11.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:11 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:39:11.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:11 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:39:11.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:11 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:39:11.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:11 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:39:11.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:11 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:39:11.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:11 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:39:11.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:11 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:39:11.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:11 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:39:11.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:11 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:39:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:11 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:39:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:11 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:39:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:11 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:39:12.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:12 smithi019 ceph-mon[24574]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:12.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:12 smithi019 ceph-mon[24574]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:39:12.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:12 smithi019 ceph-mon[24574]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:39:12.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:12 smithi019 ceph-mon[29267]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:12.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:12 smithi019 ceph-mon[29267]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:39:12.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:12 smithi019 ceph-mon[29267]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:39:12.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:12 smithi172 ceph-mon[25457]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:12.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:12 smithi172 ceph-mon[25457]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:39:12.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:12 smithi172 ceph-mon[25457]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:39:13.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:39:13.013+0000 7fcce5b20700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:39:13.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:39:13.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:39:13.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:39:13.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:39:13.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:39:13.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:39:13.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:39:13.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:39:13.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:39:13.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:39:13.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:39:13.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:39:13.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:39:13.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:39:13.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:39:13.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:39:13.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:39:13.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:39:13.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:39:13.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:39:13.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:39:13.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:39:13.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:39:13.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:39:13.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:39:13.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:39:13.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:39:13.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:39:13.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:39:13.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:39:13.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:39:13.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:39:13.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:39:13.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:39:13.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:39:13.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:39:13.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:39:13.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:39:13.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:39:13.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:39:13.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:39:13.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:39:13.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:39:13.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:39:13.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:39:13.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:39:13.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:39:13.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:39:13.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:39:13.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:39:13.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:39:13.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:39:13.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:39:13.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:39:13.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:39:13.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:39:13.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:39:13.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:39:14.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:39:13] "GET /metrics HTTP/1.1" 200 37895 "" "Prometheus/2.33.4" 2024-02-07T12:39:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:14 smithi019 ceph-mon[29267]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:39:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:39:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:39:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:39:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:39:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:39:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:39:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:39:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:39:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:39:14.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:39:14.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:39:14.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:39:14.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:39:14.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:39:14.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:39:14.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:39:14.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:39:14.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:39:14.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:39:14.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:39:14.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:39:14.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:39:14.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:39:14.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:39:14.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:39:14.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:39:14.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:39:14.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:39:14.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:39:14.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:39:14.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:39:14.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:39:14.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:39:14.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:39:14.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:39:14.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:39:14.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:39:14.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:39:14.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:39:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:39:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:39:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:39:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:39:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:39:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:39:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:39:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:39:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:39:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:39:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:39:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:39:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:39:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:39:14.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:39:14.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:39:14.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:39:14.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:39:14.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:39:14.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:14 smithi019 ceph-mon[29267]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:14.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:14 smithi019 ceph-mon[24574]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:39:14.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:39:14.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:39:14.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:39:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:39:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:39:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:39:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:39:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:39:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:39:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:39:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:39:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:39:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:39:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:39:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:39:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:39:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:39:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:39:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:39:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:39:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:39:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:39:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:39:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:39:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:39:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:39:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:39:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:39:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:39:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:39:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:39:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:39:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:39:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:39:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:39:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:39:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:39:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:39:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:39:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:39:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:39:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:39:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:39:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:39:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:39:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:39:14.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:39:14.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:39:14.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:39:14.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:39:14.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:39:14.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:39:14.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:39:14.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:39:14.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:39:14.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:39:14.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:39:14.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:39:14.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:14 smithi019 ceph-mon[24574]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:14.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:14 smithi172 ceph-mon[25457]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:39:14.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:39:14.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:39:14.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:39:14.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:39:14.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:39:14.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:39:14.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:39:14.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:39:14.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:39:14.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:39:14.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:39:14.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:39:14.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:39:14.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:39:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:39:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:39:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:39:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:39:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:39:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:39:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:39:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:39:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:39:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:39:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:39:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:39:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:39:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:39:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:39:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:39:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:39:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:39:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:39:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:39:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:39:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:39:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:39:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:39:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:39:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:39:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:39:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:39:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:39:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:39:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:39:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:39:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:39:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:39:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:39:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:39:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:39:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:39:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:39:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:39:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:39:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:39:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:39:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:39:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:14 smithi172 ceph-mon[25457]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:16.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:16 smithi019 ceph-mon[24574]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:16.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:16 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:39:16.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:16 smithi019 ceph-mon[29267]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:16.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:16 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:39:16.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:16 smithi172 ceph-mon[25457]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:16.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:16 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:39:18.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:18 smithi019 ceph-mon[29267]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:18.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:18 smithi019 ceph-mon[24574]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:18.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:18 smithi172 ceph-mon[25457]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:20.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:20 smithi019 ceph-mon[24574]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:20.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:20 smithi019 ceph-mon[29267]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:20.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:20 smithi172 ceph-mon[25457]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:21.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:21 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:39:21.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:21 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:39:21.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:21 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:39:22.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:22 smithi019 ceph-mon[24574]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:22.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:22 smithi019 ceph-mon[29267]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:22.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:22 smithi172 ceph-mon[25457]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:24.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:24 smithi019 ceph-mon[29267]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:24.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:24 smithi019 ceph-mon[24574]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:24.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:39:23] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-02-07T12:39:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:24 smithi172 ceph-mon[25457]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:26.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:26 smithi019 ceph-mon[29267]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:26.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:26 smithi019 ceph-mon[24574]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:26.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:26 smithi172 ceph-mon[25457]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:28.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:28 smithi019 ceph-mon[29267]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:28.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:28 smithi019 ceph-mon[24574]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:28.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:28 smithi172 ceph-mon[25457]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:30.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:30 smithi019 ceph-mon[24574]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:30.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:30 smithi019 ceph-mon[29267]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:30.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:30 smithi172 ceph-mon[25457]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:31.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:31 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:39:31.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:31 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:39:31.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:31 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:39:31.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:31 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:39:31.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:31 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:39:31.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:31 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:39:32.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:32 smithi019 ceph-mon[24574]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:32.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:32 smithi019 ceph-mon[29267]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:32.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:32 smithi172 ceph-mon[25457]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:34.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:34 smithi019 ceph-mon[29267]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:34.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:34 smithi019 ceph-mon[24574]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:34.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:39:33] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.33.4" 2024-02-07T12:39:34.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:34 smithi172 ceph-mon[25457]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:36.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:36 smithi019 ceph-mon[24574]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:36.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:36 smithi019 ceph-mon[29267]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:36.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:36 smithi172 ceph-mon[25457]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:38.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:38 smithi019 ceph-mon[29267]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:38.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:38 smithi019 ceph-mon[24574]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:38.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:38 smithi172 ceph-mon[25457]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:40.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:40 smithi019 ceph-mon[24574]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:40.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:40 smithi019 ceph-mon[29267]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:40.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:40 smithi172 ceph-mon[25457]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:41.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:41 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:39:41.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:41 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:39:41.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:41 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:39:42.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:42 smithi019 ceph-mon[24574]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:42.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:42 smithi019 ceph-mon[29267]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:42.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:42 smithi172 ceph-mon[25457]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:44.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:44 smithi019 ceph-mon[29267]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:44.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:44 smithi019 ceph-mon[24574]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:44.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:39:43] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.33.4" 2024-02-07T12:39:44.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:44 smithi172 ceph-mon[25457]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:46.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:46 smithi019 ceph-mon[29267]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:46.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:46 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:39:46.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:46 smithi019 ceph-mon[24574]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:46.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:46 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:39:46.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:46 smithi172 ceph-mon[25457]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:46.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:46 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:39:48.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:48 smithi019 ceph-mon[29267]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:48.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:48 smithi019 ceph-mon[24574]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:48.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:48 smithi172 ceph-mon[25457]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:50.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:50 smithi019 ceph-mon[29267]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:50.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:50 smithi019 ceph-mon[24574]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:50.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:50 smithi172 ceph-mon[25457]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:51.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:51 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:39:51.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:51 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:39:51.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:51 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:39:52.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:52 smithi019 ceph-mon[24574]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:52.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:52 smithi019 ceph-mon[29267]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:52.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:52 smithi172 ceph-mon[25457]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:54.142 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:39:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:39:53] "GET /metrics HTTP/1.1" 200 37890 "" "Prometheus/2.33.4" 2024-02-07T12:39:54.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:54 smithi019 ceph-mon[29267]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:54.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:54 smithi019 ceph-mon[24574]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:54.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:54 smithi172 ceph-mon[25457]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:56.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:56 smithi019 ceph-mon[24574]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:56.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:56 smithi019 ceph-mon[29267]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:56.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:56 smithi172 ceph-mon[25457]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:39:58.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:39:58 smithi019 ceph-mon[24574]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:58.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:39:58 smithi019 ceph-mon[29267]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:39:58.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:39:58 smithi172 ceph-mon[25457]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:00.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:00 smithi019 ceph-mon[24574]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:00.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:00 smithi019 ceph-mon[24574]: overall HEALTH_OK 2024-02-07T12:40:00.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:00 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:40:00.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:00 smithi019 ceph-mon[29267]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:00.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:00 smithi019 ceph-mon[29267]: overall HEALTH_OK 2024-02-07T12:40:00.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:00 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:40:00.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:00 smithi172 ceph-mon[25457]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:00.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:00 smithi172 ceph-mon[25457]: overall HEALTH_OK 2024-02-07T12:40:00.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:00 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:40:01.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:01 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:40:01.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:01 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:40:01.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:01 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:40:02.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:02 smithi019 ceph-mon[24574]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:02.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:02 smithi019 ceph-mon[29267]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:02.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:02 smithi172 ceph-mon[25457]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:04.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:40:03] "GET /metrics HTTP/1.1" 200 37905 "" "Prometheus/2.33.4" 2024-02-07T12:40:04.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:04 smithi019 ceph-mon[29267]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:04.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:04 smithi019 ceph-mon[24574]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:04.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:04 smithi172 ceph-mon[25457]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:06.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:06 smithi019 ceph-mon[29267]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:06.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:06 smithi019 ceph-mon[24574]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:06.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:06 smithi172 ceph-mon[25457]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:08.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:08 smithi019 ceph-mon[24574]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:08.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:08 smithi019 ceph-mon[29267]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:08.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:08 smithi172 ceph-mon[25457]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:10.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:10 smithi019 ceph-mon[29267]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:10.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:10 smithi019 ceph-mon[24574]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:10.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:10 smithi172 ceph-mon[25457]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:11.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:11 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:40:11.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:11 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:40:11.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:11 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:40:12.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:12 smithi019 ceph-mon[24574]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:12.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:12 smithi019 ceph-mon[29267]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:12.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:12 smithi172 ceph-mon[25457]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:13.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:13 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:40:13.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:13 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:40:13.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:13 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:40:14.317 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:40:13] "GET /metrics HTTP/1.1" 200 37905 "" "Prometheus/2.33.4" 2024-02-07T12:40:14.572 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:14 smithi019 ceph-mon[29267]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:14.572 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:14 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:40:14.572 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:14 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:40:14.572 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:14 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:40:14.572 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:14 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:40:14.572 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:14 smithi019 ceph-mon[29267]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:40:14.572 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:14 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:40:14.572 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:14 smithi019 ceph-mon[24574]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:14.573 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:14 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:40:14.573 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:14 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:40:14.573 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:14 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:40:14.573 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:14 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:40:14.573 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:14 smithi019 ceph-mon[24574]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:40:14.573 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:14 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:40:14.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:14 smithi172 ceph-mon[25457]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:14.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:14 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:40:14.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:14 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:40:14.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:14 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' 2024-02-07T12:40:14.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:14 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:40:14.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:14 smithi172 ceph-mon[25457]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:40:14.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:14 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:40:15.581 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-mon[29267]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:40:15.581 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-mon[29267]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:40:15.581 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:40:15.581 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-mon[24574]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:40:15.581 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-mon[24574]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:40:15.581 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:40:15.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:15 smithi172 ceph-mon[25457]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:40:15.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:15 smithi172 ceph-mon[25457]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:40:15.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:15 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:40:15.844 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:40:15.634+0000 7fcce5b20700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:40:15.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:40:15.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:40:15.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:40:15.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:40:15.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:40:15.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:40:15.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:40:15.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:40:15.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:40:15.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:40:15.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:40:15.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:40:15.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:40:15.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:40:15.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:40:15.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:40:15.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:40:15.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:40:15.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:40:15.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:40:15.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:40:15.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:40:15.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:40:15.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:40:15.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:40:15.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:40:15.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:40:15.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:40:15.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:40:15.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:40:15.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:40:15.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:40:15.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:40:15.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:40:15.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:40:15.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:40:15.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:40:15.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:40:15.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:40:15.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:40:15.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:40:15.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:40:15.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:40:15.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:40:15.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:40:15.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:40:15.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:40:15.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:40:15.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:40:15.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:40:15.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:40:15.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:40:15.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:40:15.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:40:15.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:40:15.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:40:15.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:40:15.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:15 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:40:16.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:16 smithi019 ceph-mon[29267]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:16.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:16 smithi019 ceph-mon[24574]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:16.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:16 smithi172 ceph-mon[25457]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:17.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:17 smithi019 ceph-mon[24574]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:40:17.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:40:17.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:40:17.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:40:17.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:40:17.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:40:17.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:40:17.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:40:17.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:40:17.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:40:17.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:40:17.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:40:17.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:40:17.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:40:17.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:40:17.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:40:17.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:40:17.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:40:17.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:40:17.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:40:17.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:40:17.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:40:17.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:40:17.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:40:17.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:40:17.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:40:17.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:40:17.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:40:17.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:40:17.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:40:17.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:40:17.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:40:17.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:40:17.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:40:17.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:40:17.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:40:17.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:40:17.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:40:17.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:40:17.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:40:17.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:40:17.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:40:17.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:40:17.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:40:17.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:40:17.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:40:17.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:40:17.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:40:17.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:40:17.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:40:17.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:40:17.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:40:17.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:40:17.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:40:17.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:40:17.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:40:17.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:40:17.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:40:17.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:40:17.600 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:17 smithi019 ceph-mon[29267]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:40:17.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:40:17.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:40:17.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:40:17.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:40:17.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:40:17.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:40:17.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:40:17.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:40:17.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:40:17.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:40:17.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:40:17.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:40:17.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:40:17.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:40:17.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:40:17.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:40:17.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:40:17.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:40:17.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:40:17.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:40:17.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:40:17.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:40:17.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:40:17.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:40:17.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:40:17.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:40:17.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:40:17.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:40:17.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:40:17.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:40:17.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:40:17.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:40:17.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:40:17.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:40:17.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:40:17.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:40:17.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:40:17.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:40:17.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:40:17.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:40:17.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:40:17.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:40:17.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:40:17.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:40:17.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:40:17.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:40:17.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:40:17.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:40:17.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:40:17.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:40:17.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:40:17.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:40:17.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:40:17.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:40:17.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:40:17.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:40:17.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:40:17.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:40:17.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:17 smithi172 ceph-mon[25457]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:40:17.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:40:17.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:40:17.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:40:17.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:40:17.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:40:17.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:40:17.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:40:17.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:40:17.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:40:17.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:40:17.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:40:17.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:40:17.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:40:17.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:40:17.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:40:17.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:40:17.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:40:17.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:40:17.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:40:17.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:40:17.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:40:17.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:40:17.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:40:17.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:40:17.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:40:17.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:40:17.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:40:17.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:40:17.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:40:17.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:40:17.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:40:17.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:40:17.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:40:17.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:40:17.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:40:17.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:40:17.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:40:17.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:40:17.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:40:17.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:40:17.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:40:17.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:40:17.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:40:17.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:40:17.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:40:17.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:40:17.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:40:17.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:40:17.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:40:17.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:40:17.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:40:17.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:40:17.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:40:17.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:40:17.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:40:17.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:40:17.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:40:17.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:40:18.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:18 smithi019 ceph-mon[29267]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:18.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:18 smithi019 ceph-mon[24574]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:18.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:18 smithi172 ceph-mon[25457]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:20.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:20 smithi019 ceph-mon[29267]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:20.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:20 smithi019 ceph-mon[24574]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:20.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:20 smithi172 ceph-mon[25457]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:21.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:21 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:40:21.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:21 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:40:21.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:21 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:40:22.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:22 smithi172 ceph-mon[25457]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:22.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:22 smithi019 ceph-mon[29267]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:22.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:22 smithi019 ceph-mon[24574]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:24.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:24 smithi019 ceph-mon[29267]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:24.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:24 smithi019 ceph-mon[24574]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:24.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:40:23] "GET /metrics HTTP/1.1" 200 37904 "" "Prometheus/2.33.4" 2024-02-07T12:40:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:24 smithi172 ceph-mon[25457]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:26.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:26 smithi019 ceph-mon[24574]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:26.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:26 smithi019 ceph-mon[29267]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:26.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:26 smithi172 ceph-mon[25457]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:28.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:28 smithi019 ceph-mon[29267]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:28.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:28 smithi019 ceph-mon[24574]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:28.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:28 smithi172 ceph-mon[25457]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:30.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:30 smithi019 ceph-mon[29267]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:30.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:30 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:40:30.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:30 smithi019 ceph-mon[24574]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:30.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:30 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:40:30.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:30 smithi172 ceph-mon[25457]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:30.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:30 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:40:31.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:31 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:40:31.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:31 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:40:31.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:31 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:40:32.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:32 smithi019 ceph-mon[24574]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:32.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:32 smithi019 ceph-mon[29267]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:32.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:32 smithi172 ceph-mon[25457]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:34.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:40:33] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.33.4" 2024-02-07T12:40:34.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:34 smithi019 ceph-mon[29267]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:34.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:34 smithi019 ceph-mon[24574]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:34.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:34 smithi172 ceph-mon[25457]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:36.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:36 smithi019 ceph-mon[24574]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:36.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:36 smithi019 ceph-mon[29267]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:36.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:36 smithi172 ceph-mon[25457]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:38.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:38 smithi019 ceph-mon[24574]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:38.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:38 smithi019 ceph-mon[29267]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:38.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:38 smithi172 ceph-mon[25457]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:40.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:40 smithi019 ceph-mon[29267]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:40.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:40 smithi019 ceph-mon[24574]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:40.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:40 smithi172 ceph-mon[25457]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:41.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:41 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:40:41.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:41 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:40:41.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:41 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:40:42.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:42 smithi019 ceph-mon[29267]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:42.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:42 smithi019 ceph-mon[24574]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:42.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:42 smithi172 ceph-mon[25457]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:44.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:44 smithi019 ceph-mon[24574]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:44.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:44 smithi019 ceph-mon[29267]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:44.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:40:43] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.33.4" 2024-02-07T12:40:44.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:44 smithi172 ceph-mon[25457]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:45.601 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:45 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:40:45.601 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:45 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:40:45.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:45 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:40:46.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:46 smithi172 ceph-mon[25457]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:46.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:46 smithi019 ceph-mon[29267]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:46.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:46 smithi019 ceph-mon[24574]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:48.190 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch ps' 2024-02-07T12:40:48.509 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:48 smithi019 ceph-mon[29267]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:48.509 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:48 smithi019 ceph-mon[24574]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:48.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:48 smithi172 ceph-mon[25457]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:50.301 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:40:50.301 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (9m) 2m ago 16m 24.1M - 0.23.0 ba2b418f427c dfaf78b29ab5 2024-02-07T12:40:50.301 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (8m) 2m ago 16m 49.5M - 8.3.5 dad864ee21e9 e753e96fa4f0 2024-02-07T12:40:50.301 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 running (9m) 2m ago 16m 56.8M - 3.5 e1d6a67b021e 8072de108b37 2024-02-07T12:40:50.301 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283 running (11m) 2m ago 21m 447M - 19.0.0-1252-g44458cce 79957b372307 97644267e529 2024-02-07T12:40:50.301 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (6m) 2m ago 23m 476M - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:40:50.301 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (23m) 2m ago 23m 64.2M 2048M 17.2.0 e1d6a67b021e f3a8a050cf1d 2024-02-07T12:40:50.301 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (21m) 2m ago 21m 45.5M 2048M 17.2.0 e1d6a67b021e 7d62c7c4d075 2024-02-07T12:40:50.301 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (21m) 2m ago 21m 47.8M 2048M 17.2.0 e1d6a67b021e b9825d0db32d 2024-02-07T12:40:50.301 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (9m) 2m ago 17m 18.3M - 1.3.1 1dbe0e931976 9868df14bcc9 2024-02-07T12:40:50.301 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (8m) 2m ago 17m 17.6M - 1.3.1 1dbe0e931976 e066f1871320 2024-02-07T12:40:50.301 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (21m) 2m ago 21m 52.9M 2993M 17.2.0 e1d6a67b021e 87f2560a9b15 2024-02-07T12:40:50.302 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (20m) 2m ago 20m 55.5M 2993M 17.2.0 e1d6a67b021e 0f9a190f0e0d 2024-02-07T12:40:50.302 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (20m) 2m ago 20m 52.2M 2993M 17.2.0 e1d6a67b021e 7db94f25a1aa 2024-02-07T12:40:50.302 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (19m) 2m ago 19m 52.5M 2993M 17.2.0 e1d6a67b021e 0b6b674188ed 2024-02-07T12:40:50.302 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (19m) 2m ago 19m 53.5M 3248M 17.2.0 e1d6a67b021e df72cfd69b3b 2024-02-07T12:40:50.302 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (19m) 2m ago 19m 51.6M 3248M 17.2.0 e1d6a67b021e 3bdb22c07537 2024-02-07T12:40:50.302 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (18m) 2m ago 18m 50.1M 3248M 17.2.0 e1d6a67b021e 7859b651e0d9 2024-02-07T12:40:50.302 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (18m) 2m ago 18m 54.6M 3248M 17.2.0 e1d6a67b021e b301e7c75d73 2024-02-07T12:40:50.302 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (2m) 2m ago 17m 46.8M - 2.33.4 514e6a882f6e 7371d5c3aec0 2024-02-07T12:40:50.302 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (16m) 2m ago 16m 75.5M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:40:50.302 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (16m) 2m ago 16m 75.1M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:40:50.573 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:50 smithi019 ceph-mon[24574]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:50.573 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:50 smithi019 ceph-mon[29267]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:50.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:50 smithi172 ceph-mon[25457]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:50.973 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph versions' 2024-02-07T12:40:51.536 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:51 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:40:51.536 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:51 smithi019 ceph-mon[24574]: from='client.15255 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:40:51.537 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:51 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:40:51.537 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:51 smithi019 ceph-mon[29267]: from='client.15255 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:40:51.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:51 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:40:51.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:51 smithi172 ceph-mon[25457]: from='client.15255 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:40:52.416 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:52 smithi019 ceph-mon[29267]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:52.416 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:52 smithi019 ceph-mon[24574]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:52.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:52 smithi172 ceph-mon[25457]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:52.886 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:40:52.887 INFO:teuthology.orchestra.run.smithi019.stdout: "mon": { 2024-02-07T12:40:52.887 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-02-07T12:40:52.887 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:40:52.887 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr": { 2024-02-07T12:40:52.887 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:40:52.887 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:40:52.887 INFO:teuthology.orchestra.run.smithi019.stdout: "osd": { 2024-02-07T12:40:52.887 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-02-07T12:40:52.887 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:40:52.887 INFO:teuthology.orchestra.run.smithi019.stdout: "mds": {}, 2024-02-07T12:40:52.887 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw": { 2024-02-07T12:40:52.887 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:40:52.887 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:40:52.888 INFO:teuthology.orchestra.run.smithi019.stdout: "overall": { 2024-02-07T12:40:52.888 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-02-07T12:40:52.888 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:40:52.888 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:40:52.888 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:40:53.489 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph -s' 2024-02-07T12:40:53.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:53 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2305118073' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:40:53.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:53 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2305118073' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:40:53.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:53 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2305118073' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:40:54.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:40:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:40:53] "GET /metrics HTTP/1.1" 200 37902 "" "Prometheus/2.33.4" 2024-02-07T12:40:54.725 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:54 smithi019 ceph-mon[24574]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:54.725 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:54 smithi019 ceph-mon[29267]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:54.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:54 smithi172 ceph-mon[25457]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:55.479 INFO:teuthology.orchestra.run.smithi019.stdout: cluster: 2024-02-07T12:40:55.479 INFO:teuthology.orchestra.run.smithi019.stdout: id: ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:40:55.479 INFO:teuthology.orchestra.run.smithi019.stdout: health: HEALTH_OK 2024-02-07T12:40:55.479 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:40:55.479 INFO:teuthology.orchestra.run.smithi019.stdout: services: 2024-02-07T12:40:55.479 INFO:teuthology.orchestra.run.smithi019.stdout: mon: 3 daemons, quorum a,c,b (age 21m) 2024-02-07T12:40:55.480 INFO:teuthology.orchestra.run.smithi019.stdout: mgr: y(active, since 3m), standbys: x 2024-02-07T12:40:55.480 INFO:teuthology.orchestra.run.smithi019.stdout: osd: 8 osds: 8 up (since 18m), 8 in (since 18m) 2024-02-07T12:40:55.480 INFO:teuthology.orchestra.run.smithi019.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-02-07T12:40:55.483 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:40:55.483 INFO:teuthology.orchestra.run.smithi019.stdout: data: 2024-02-07T12:40:55.483 INFO:teuthology.orchestra.run.smithi019.stdout: pools: 6 pools, 161 pgs 2024-02-07T12:40:55.484 INFO:teuthology.orchestra.run.smithi019.stdout: objects: 209 objects, 585 KiB 2024-02-07T12:40:55.484 INFO:teuthology.orchestra.run.smithi019.stdout: usage: 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:40:55.484 INFO:teuthology.orchestra.run.smithi019.stdout: pgs: 161 active+clean 2024-02-07T12:40:55.484 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:40:55.484 INFO:teuthology.orchestra.run.smithi019.stdout: io: 2024-02-07T12:40:55.484 INFO:teuthology.orchestra.run.smithi019.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-02-07T12:40:55.484 INFO:teuthology.orchestra.run.smithi019.stdout: 2024-02-07T12:40:56.115 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph health detail' 2024-02-07T12:40:56.651 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:56 smithi019 ceph-mon[29267]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:56.651 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:56 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3706459251' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-02-07T12:40:56.652 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:56 smithi019 ceph-mon[24574]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:56.652 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:56 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3706459251' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-02-07T12:40:56.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:56 smithi172 ceph-mon[25457]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:40:56.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:56 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3706459251' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-02-07T12:40:58.169 INFO:teuthology.orchestra.run.smithi019.stdout:HEALTH_OK 2024-02-07T12:40:58.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:58 smithi019 ceph-mon[24574]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:58.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:40:58 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2345395495' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:40:58.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:58 smithi019 ceph-mon[29267]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:58.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:40:58 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2345395495' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:40:58.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:58 smithi172 ceph-mon[25457]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:40:58.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:40:58 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2345395495' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:40:58.885 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-02-07T12:41:00.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:00 smithi019 ceph-mon[24574]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:00.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:00 smithi019 ceph-mon[24574]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:41:00.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:00 smithi019 ceph-mon[29267]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:00.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:00 smithi019 ceph-mon[29267]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:41:00.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:00 smithi172 ceph-mon[25457]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:00.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:00 smithi172 ceph-mon[25457]: from='mgr.15165 172.21.15.19:0/271108764' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:41:00.925 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:41:01.519 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:01 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:41:01.519 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:01 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2158225002' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:41:01.519 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:01 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:41:01.519 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:01 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2158225002' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:41:01.549 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph mgr fail' 2024-02-07T12:41:01.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:01 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:41:01.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:01 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2158225002' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:41:02.460 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:02 smithi019 ceph-mon[24574]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:41:02.460 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:02 smithi019 ceph-mon[29267]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:41:02.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:02 smithi172 ceph-mon[25457]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:41:04.167 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:41:03] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.33.4" 2024-02-07T12:41:04.168 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:04 smithi019 ceph-mon[29267]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:04.168 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:04 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3746366844' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-02-07T12:41:04.168 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:04 smithi019 ceph-mon[29267]: osdmap e103: 8 total, 8 up, 8 in 2024-02-07T12:41:04.168 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:04 smithi019 ceph-mon[29267]: Standby manager daemon y started 2024-02-07T12:41:04.168 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:04 smithi019 ceph-mon[24574]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:04.169 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:04 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3746366844' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-02-07T12:41:04.169 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:04 smithi019 ceph-mon[24574]: osdmap e103: 8 total, 8 up, 8 in 2024-02-07T12:41:04.169 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:04 smithi019 ceph-mon[24574]: Standby manager daemon y started 2024-02-07T12:41:04.449 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:04 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:41:04.163+0000 7fcd533e9700 -1 mgr handle_mgr_map I was active but no longer am 2024-02-07T12:41:04.449 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:04 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ignoring --setuser ceph since I am not root 2024-02-07T12:41:04.449 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:04 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ignoring --setgroup ceph since I am not root 2024-02-07T12:41:04.449 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:04 smithi019 ceph-mgr[61877]: -- 172.21.15.19:0/2253770153 <== mon.1 v2:172.21.15.19:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x556bb5c11520 con 0x556bb5be6c00 2024-02-07T12:41:04.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:04 smithi172 ceph-mon[25457]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:04.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:04 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3746366844' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-02-07T12:41:04.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:04 smithi172 ceph-mon[25457]: osdmap e103: 8 total, 8 up, 8 in 2024-02-07T12:41:04.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:04 smithi172 ceph-mon[25457]: Standby manager daemon y started 2024-02-07T12:41:04.620 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:41:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:41:04] ENGINE Bus STOPPING 2024-02-07T12:41:04.620 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:41:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:41:04] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-02-07T12:41:04.620 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:41:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:41:04] ENGINE Bus STOPPED 2024-02-07T12:41:04.620 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:41:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:41:04] ENGINE Bus STARTING 2024-02-07T12:41:04.620 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:41:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:41:04] ENGINE Serving on http://:::9283 2024-02-07T12:41:04.620 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:41:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:41:04] ENGINE Bus STARTED 2024-02-07T12:41:04.829 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'sleep 180' 2024-02-07T12:41:05.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:04 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:41:04.781+0000 7effdac41200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-02-07T12:41:05.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:04 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:41:04.907+0000 7effdac41200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-02-07T12:41:05.465 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3746366844' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-02-07T12:41:05.465 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:41:05.465 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[29267]: mgrmap e34: x(active, starting, since 0.718278s), standbys: y 2024-02-07T12:41:05.466 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:41:05.466 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:41:05.466 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-02-07T12:41:05.466 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-02-07T12:41:05.466 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:41:05.466 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:41:05.466 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:41:05.466 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:41:05.466 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:41:05.466 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:41:05.466 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:41:05.466 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:41:05.466 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:41:05.466 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:41:05.466 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:41:05.467 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[29267]: Manager daemon x is now available 2024-02-07T12:41:05.467 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:41:05.467 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:41:05.467 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:41:05.467 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:41:05.467 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:41:05.467 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-02-07T12:41:05.467 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-02-07T12:41:05.467 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3746366844' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-02-07T12:41:05.467 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:41:05.467 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[24574]: mgrmap e34: x(active, starting, since 0.718278s), standbys: y 2024-02-07T12:41:05.467 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:41:05.467 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:41:05.468 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-02-07T12:41:05.468 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-02-07T12:41:05.468 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:41:05.468 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:41:05.468 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:41:05.468 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:41:05.468 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:41:05.468 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:41:05.468 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:41:05.468 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:41:05.468 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:41:05.468 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:41:05.468 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:41:05.468 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[24574]: Manager daemon x is now available 2024-02-07T12:41:05.468 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:41:05.469 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:41:05.469 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:41:05.469 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:41:05.469 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:41:05.469 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-02-07T12:41:05.469 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-02-07T12:41:05.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:05 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3746366844' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-02-07T12:41:05.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:05 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:41:05.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:05 smithi172 ceph-mon[25457]: mgrmap e34: x(active, starting, since 0.718278s), standbys: y 2024-02-07T12:41:05.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:05 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:41:05.470 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:05 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:41:05.470 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:05 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-02-07T12:41:05.470 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:05 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-02-07T12:41:05.470 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:05 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:41:05.470 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:05 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:41:05.470 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:05 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:41:05.470 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:05 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:41:05.470 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:05 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:41:05.470 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:05 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:41:05.470 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:05 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:41:05.470 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:05 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:41:05.470 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:05 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:41:05.470 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:05 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:41:05.470 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:05 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:41:05.470 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:05 smithi172 ceph-mon[25457]: Manager daemon x is now available 2024-02-07T12:41:05.470 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:05 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:41:05.471 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:05 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:41:05.471 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:05 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:41:05.471 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:05 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:41:05.471 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:05 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:41:05.471 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:05 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-02-07T12:41:05.471 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:05 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-02-07T12:41:05.720 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:41:05.619+0000 7effdac41200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-02-07T12:41:06.053 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:41:05.716+0000 7effdac41200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-02-07T12:41:06.054 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:41:05.867+0000 7effdac41200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-02-07T12:41:06.054 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:05 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:41:05.965+0000 7effdac41200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-02-07T12:41:06.054 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:06 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:41:06.050+0000 7effdac41200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-02-07T12:41:06.339 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:06 smithi019 ceph-mon[29267]: mgrmap e35: x(active, since 1.72383s), standbys: y 2024-02-07T12:41:06.340 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:06 smithi019 ceph-mon[29267]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:41:06.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:06 smithi019 ceph-mon[24574]: mgrmap e35: x(active, since 1.72383s), standbys: y 2024-02-07T12:41:06.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:06 smithi019 ceph-mon[24574]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:41:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:06 smithi172 ceph-mon[25457]: mgrmap e35: x(active, since 1.72383s), standbys: y 2024-02-07T12:41:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:06 smithi172 ceph-mon[25457]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:41:06.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:06 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:41:06.336+0000 7effdac41200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-02-07T12:41:06.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:06 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:41:06.453+0000 7effdac41200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-02-07T12:41:06.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:06 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:41:06.542+0000 7effdac41200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-02-07T12:41:06.961 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:06 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:41:06.643+0000 7effdac41200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-02-07T12:41:06.961 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:06 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:41:06.824+0000 7effdac41200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-02-07T12:41:07.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:07 smithi019 ceph-mon[29267]: [07/Feb/2024:12:41:05] ENGINE Bus STARTING 2024-02-07T12:41:07.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:07 smithi019 ceph-mon[29267]: [07/Feb/2024:12:41:05] ENGINE Serving on https://172.21.15.172:7150 2024-02-07T12:41:07.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:07 smithi019 ceph-mon[29267]: [07/Feb/2024:12:41:05] ENGINE Serving on http://172.21.15.172:8765 2024-02-07T12:41:07.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:07 smithi019 ceph-mon[29267]: [07/Feb/2024:12:41:05] ENGINE Bus STARTED 2024-02-07T12:41:07.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:07 smithi019 ceph-mon[29267]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:41:07.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:07 smithi019 ceph-mon[29267]: mgrmap e36: x(active, since 2s), standbys: y 2024-02-07T12:41:07.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:07 smithi019 ceph-mon[24574]: [07/Feb/2024:12:41:05] ENGINE Bus STARTING 2024-02-07T12:41:07.347 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:07 smithi019 ceph-mon[24574]: [07/Feb/2024:12:41:05] ENGINE Serving on https://172.21.15.172:7150 2024-02-07T12:41:07.347 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:07 smithi019 ceph-mon[24574]: [07/Feb/2024:12:41:05] ENGINE Serving on http://172.21.15.172:8765 2024-02-07T12:41:07.347 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:07 smithi019 ceph-mon[24574]: [07/Feb/2024:12:41:05] ENGINE Bus STARTED 2024-02-07T12:41:07.347 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:07 smithi019 ceph-mon[24574]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:41:07.347 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:07 smithi019 ceph-mon[24574]: mgrmap e36: x(active, since 2s), standbys: y 2024-02-07T12:41:07.597 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:07 smithi172 ceph-mon[25457]: [07/Feb/2024:12:41:05] ENGINE Bus STARTING 2024-02-07T12:41:07.597 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:07 smithi172 ceph-mon[25457]: [07/Feb/2024:12:41:05] ENGINE Serving on https://172.21.15.172:7150 2024-02-07T12:41:07.597 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:07 smithi172 ceph-mon[25457]: [07/Feb/2024:12:41:05] ENGINE Serving on http://172.21.15.172:8765 2024-02-07T12:41:07.597 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:07 smithi172 ceph-mon[25457]: [07/Feb/2024:12:41:05] ENGINE Bus STARTED 2024-02-07T12:41:07.597 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:07 smithi172 ceph-mon[25457]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:41:07.597 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:07 smithi172 ceph-mon[25457]: mgrmap e36: x(active, since 2s), standbys: y 2024-02-07T12:41:08.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:41:07.853+0000 7effdac41200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-02-07T12:41:08.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:41:07.989+0000 7effdac41200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-02-07T12:41:08.653 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:08 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:41:08.366+0000 7effdac41200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-02-07T12:41:08.653 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:08 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:41:08.460+0000 7effdac41200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-02-07T12:41:09.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:08 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:41:08.669+0000 7effdac41200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:41:09.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:08 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:41:08.862+0000 7effdac41200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-02-07T12:41:09.443 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:41:09.169+0000 7effdac41200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:41:09.443 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:09 smithi019 ceph-mon[24574]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:41:09.443 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:09 smithi019 ceph-mon[24574]: mgrmap e37: x(active, since 4s), standbys: y 2024-02-07T12:41:09.443 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:09 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:09.443 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:09 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:09.443 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:09 smithi019 ceph-mon[29267]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:41:09.444 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:09 smithi019 ceph-mon[29267]: mgrmap e37: x(active, since 4s), standbys: y 2024-02-07T12:41:09.444 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:09 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:09.444 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:09 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:09.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:09 smithi172 ceph-mon[25457]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:41:09.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:09 smithi172 ceph-mon[25457]: mgrmap e37: x(active, since 4s), standbys: y 2024-02-07T12:41:09.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:09 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:09.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:09 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:09.705 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:41:09.445+0000 7effdac41200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-02-07T12:41:09.706 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:41:09.702+0000 7effdac41200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-02-07T12:41:10.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:41:10.563+0000 7effdac41200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-02-07T12:41:10.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:10 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:10.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:10 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:10.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:10 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:41:10.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:10 smithi019 ceph-mon[29267]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:41:10.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:10 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:10.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:10 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:10.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:10 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:10.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:10 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:10.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:10 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:41:10.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:10 smithi019 ceph-mon[24574]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:41:10.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:10 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:10.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:10 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:11.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:10 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:11.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:10 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:11.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:10 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:41:11.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:10 smithi172 ceph-mon[25457]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:41:11.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:10 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:11.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:10 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:11.547 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:41:11.201+0000 7effdac41200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-02-07T12:41:11.547 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:41:11.309+0000 7effdac41200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-02-07T12:41:11.547 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:41:11.543+0000 7effdac41200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-02-07T12:41:11.844 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: [07/Feb/2024:12:41:11] ENGINE Bus STARTING 2024-02-07T12:41:11.844 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CherryPy Checker: 2024-02-07T12:41:11.844 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: The Application mounted at '' has an empty config. 2024-02-07T12:41:11.844 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:41:11.844 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: [07/Feb/2024:12:41:11] ENGINE Serving on http://:::9283 2024-02-07T12:41:11.844 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: [07/Feb/2024:12:41:11] ENGINE Bus STARTED 2024-02-07T12:41:11.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-mon[29267]: Standby manager daemon y restarted 2024-02-07T12:41:11.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-mon[29267]: Standby manager daemon y started 2024-02-07T12:41:11.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-02-07T12:41:11.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:41:11.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-02-07T12:41:11.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:41:11.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:11.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:11.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:41:11.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:41:11.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-mon[24574]: Standby manager daemon y restarted 2024-02-07T12:41:11.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-mon[24574]: Standby manager daemon y started 2024-02-07T12:41:11.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-02-07T12:41:11.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:41:11.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-02-07T12:41:11.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:41:11.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:11.847 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:11.847 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:41:11.847 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:11 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:41:12.048 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:11 smithi172 ceph-mon[25457]: Standby manager daemon y restarted 2024-02-07T12:41:12.048 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:11 smithi172 ceph-mon[25457]: Standby manager daemon y started 2024-02-07T12:41:12.048 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:11 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-02-07T12:41:12.048 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:11 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:41:12.048 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:11 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-02-07T12:41:12.049 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:11 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:41:12.049 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:11 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:12.049 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:11 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:12.049 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:11 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:41:12.049 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:11 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:41:13.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[24574]: Updating smithi019:/etc/ceph/ceph.conf 2024-02-07T12:41:13.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[24574]: Updating smithi172:/etc/ceph/ceph.conf 2024-02-07T12:41:13.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[24574]: mgrmap e38: x(active, since 8s), standbys: y 2024-02-07T12:41:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[24574]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:41:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[24574]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:41:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[24574]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:41:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[24574]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:41:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[24574]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-02-07T12:41:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[24574]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:41:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:13.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[29267]: Updating smithi019:/etc/ceph/ceph.conf 2024-02-07T12:41:13.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[29267]: Updating smithi172:/etc/ceph/ceph.conf 2024-02-07T12:41:13.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[29267]: mgrmap e38: x(active, since 8s), standbys: y 2024-02-07T12:41:13.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[29267]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:41:13.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[29267]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:41:13.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[29267]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:41:13.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[29267]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:41:13.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[29267]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-02-07T12:41:13.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[29267]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:41:13.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:13.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:13.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:13.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:13.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:12 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:13.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:12 smithi172 ceph-mon[25457]: Updating smithi019:/etc/ceph/ceph.conf 2024-02-07T12:41:13.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:12 smithi172 ceph-mon[25457]: Updating smithi172:/etc/ceph/ceph.conf 2024-02-07T12:41:13.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:12 smithi172 ceph-mon[25457]: mgrmap e38: x(active, since 8s), standbys: y 2024-02-07T12:41:13.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:12 smithi172 ceph-mon[25457]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:41:13.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:12 smithi172 ceph-mon[25457]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:41:13.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:12 smithi172 ceph-mon[25457]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:41:13.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:12 smithi172 ceph-mon[25457]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:41:13.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:12 smithi172 ceph-mon[25457]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-02-07T12:41:13.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:12 smithi172 ceph-mon[25457]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:41:13.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:12 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:13.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:12 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:13.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:12 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:13.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:12 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:13.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:12 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:13.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:13 smithi172 ceph-mon[25457]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:41:13.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:13 smithi172 ceph-mon[25457]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 16 op/s 2024-02-07T12:41:13.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:13 smithi172 ceph-mon[25457]: Reconfiguring prometheus.a (dependencies changed)... 2024-02-07T12:41:13.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:13 smithi172 ceph-mon[25457]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-02-07T12:41:13.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:13 smithi172 ceph-mon[25457]: Reconfiguring daemon prometheus.a on smithi172 2024-02-07T12:41:14.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:41:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:41:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-02-07T12:41:14.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:13 smithi019 ceph-mon[29267]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:41:14.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:13 smithi019 ceph-mon[29267]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 16 op/s 2024-02-07T12:41:14.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:13 smithi019 ceph-mon[29267]: Reconfiguring prometheus.a (dependencies changed)... 2024-02-07T12:41:14.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:13 smithi019 ceph-mon[29267]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-02-07T12:41:14.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:13 smithi019 ceph-mon[29267]: Reconfiguring daemon prometheus.a on smithi172 2024-02-07T12:41:14.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:13 smithi019 ceph-mon[24574]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:41:14.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:13 smithi019 ceph-mon[24574]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 16 op/s 2024-02-07T12:41:14.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:13 smithi019 ceph-mon[24574]: Reconfiguring prometheus.a (dependencies changed)... 2024-02-07T12:41:14.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:13 smithi019 ceph-mon[24574]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-02-07T12:41:14.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:13 smithi019 ceph-mon[24574]: Reconfiguring daemon prometheus.a on smithi172 2024-02-07T12:41:16.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:15 smithi019 ceph-mon[24574]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-02-07T12:41:16.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:15 smithi019 ceph-mon[29267]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-02-07T12:41:16.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:15 smithi172 ceph-mon[25457]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-02-07T12:41:16.576 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:16 smithi172 systemd[1]: Stopping Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:41:16.576 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:16 smithi172 bash[57304]: Error: no container with name or ID "ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus.a" found: no such container 2024-02-07T12:41:16.576 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:16 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:41:16.331Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-02-07T12:41:16.577 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:16 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:41:16.332Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-02-07T12:41:16.577 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:16 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:41:16.332Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-02-07T12:41:16.577 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:16 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:41:16.332Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-02-07T12:41:16.577 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:16 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:41:16.332Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-02-07T12:41:16.577 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:16 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:41:16.332Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-02-07T12:41:16.577 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:16 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:41:16.332Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-02-07T12:41:16.577 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:16 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:41:16.332Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-02-07T12:41:16.577 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:16 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:41:16.332Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-02-07T12:41:16.577 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:16 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:41:16.332Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-02-07T12:41:16.577 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:16 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:41:16.332Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-02-07T12:41:16.577 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:16 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[54370]: ts=2024-02-07T12:41:16.332Z caller=main.go:1066 level=info msg="See you next time!" 2024-02-07T12:41:16.577 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:16 smithi172 podman[57317]: 2024-02-07 12:41:16.342225659 +0000 UTC m=+0.127400458 container died 7371d5c3aec06aed9ba002c759b308247210d2c2e63f7df565282288abbb1fdb (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:41:16.858 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:16 smithi172 podman[57317]: 2024-02-07 12:41:16.672763177 +0000 UTC m=+0.457937970 container cleanup 7371d5c3aec06aed9ba002c759b308247210d2c2e63f7df565282288abbb1fdb (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:41:16.859 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:16 smithi172 bash[57317]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a 2024-02-07T12:41:17.120 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:16 smithi172 podman[57331]: 2024-02-07 12:41:16.857230527 +0000 UTC m=+0.516620086 container remove 7371d5c3aec06aed9ba002c759b308247210d2c2e63f7df565282288abbb1fdb (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:41:17.120 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:16 smithi172 bash[57345]: Error: no container with name or ID "ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus.a" found: no such container 2024-02-07T12:41:17.120 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:16 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@prometheus.a.service: Deactivated successfully. 2024-02-07T12:41:17.120 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:16 smithi172 systemd[1]: Stopped Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:41:17.120 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:16 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@prometheus.a.service: Consumed 1.295s CPU time. 2024-02-07T12:41:17.120 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:16 smithi172 systemd[1]: Starting Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:41:17.374 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 podman[57414]: 2024-02-07 12:41:17.026415729 +0000 UTC m=+0.020183642 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-02-07T12:41:17.374 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 podman[57414]: 2024-02-07 12:41:17.124688889 +0000 UTC m=+0.118456798 container create 6b7568bd4b22de5ab4b12549d05e2479331daa41fdac2c5c0545065838b6dabd (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:41:17.375 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 podman[57414]: 2024-02-07 12:41:17.343799417 +0000 UTC m=+0.337567338 container init 6b7568bd4b22de5ab4b12549d05e2479331daa41fdac2c5c0545065838b6dabd (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:41:17.375 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 podman[57414]: 2024-02-07 12:41:17.346416186 +0000 UTC m=+0.340184106 container start 6b7568bd4b22de5ab4b12549d05e2479331daa41fdac2c5c0545065838b6dabd (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:41:17.375 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:41:17.369Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-02-07T12:41:17.375 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:41:17.369Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-02-07T12:41:17.375 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:41:17.369Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-02-07T12:41:17.375 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:41:17.369Z caller=main.go:518 level=info host_details="(Linux 5.14.0-418.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Mon Feb 5 23:17:48 UTC 2024 x86_64 smithi172 (none))" 2024-02-07T12:41:17.375 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:41:17.369Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-02-07T12:41:17.375 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:41:17.369Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-02-07T12:41:17.375 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:41:17.372Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-02-07T12:41:17.817 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:41:17.372Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-02-07T12:41:17.818 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:41:17.373Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-02-07T12:41:17.818 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:41:17.375Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-02-07T12:41:17.818 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:41:17.375Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.865µs 2024-02-07T12:41:17.818 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:41:17.375Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-02-07T12:41:17.818 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 bash[57414]: 6b7568bd4b22de5ab4b12549d05e2479331daa41fdac2c5c0545065838b6dabd 2024-02-07T12:41:17.818 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 systemd[1]: Started Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:41:17.818 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:41:17.595Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-02-07T12:41:18.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:17 smithi019 ceph-mon[24574]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-02-07T12:41:18.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:17 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:18.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:17 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:18.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:17 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:41:18.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:17 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:41:18.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:17 smithi019 ceph-mon[29267]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-02-07T12:41:18.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:17 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:18.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:17 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:18.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:17 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:41:18.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:17 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:41:18.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-mon[25457]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-02-07T12:41:18.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:18.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:18.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:41:18.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:41:18.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:41:17.833Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-02-07T12:41:18.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:41:17.892Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-02-07T12:41:18.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:41:17.950Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-02-07T12:41:18.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:41:17.950Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-02-07T12:41:18.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:41:17.950Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=50.25µs wal_replay_duration=575.467006ms total_replay_duration=575.530308ms 2024-02-07T12:41:18.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:41:17.954Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-02-07T12:41:18.120 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:41:17.954Z caller=main.go:947 level=info msg="TSDB started" 2024-02-07T12:41:18.120 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:41:17.954Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-02-07T12:41:18.120 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:41:17.971Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.293707ms db_storage=852ns remote_storage=1.392µs web_handler=395ns query_engine=698ns scrape=159.401µs scrape_sd=114.995µs notify=10.819µs notify_sd=12.161µs rules=16.738791ms 2024-02-07T12:41:18.120 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:41:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:41:17.971Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-02-07T12:41:19.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:19 smithi172 ceph-mon[25457]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:41:19.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:19 smithi019 ceph-mon[29267]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:41:19.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:19 smithi019 ceph-mon[24574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:41:20.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:20 smithi019 ceph-mon[29267]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-02-07T12:41:20.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:20 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:20.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:20 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:41:20.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:20 smithi019 ceph-mon[24574]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-02-07T12:41:20.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:20 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:20.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:20 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:41:20.599 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:20 smithi172 ceph-mon[25457]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-02-07T12:41:20.599 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:20 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:20.600 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:20 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:41:21.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:21 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:41:21.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:21 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:41:21.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:21 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:41:22.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:22 smithi019 ceph-mon[29267]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-02-07T12:41:22.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:22 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:22.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:22 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:22.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:22 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:41:22.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:22 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:41:22.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:22 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:22.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:22 smithi019 ceph-mon[24574]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-02-07T12:41:22.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:22 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:22.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:22 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:22.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:22 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:41:22.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:22 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:41:22.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:22 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:22.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:22 smithi172 ceph-mon[25457]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-02-07T12:41:22.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:22 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:22.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:22 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:22.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:22 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:41:22.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:22 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:41:22.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:22 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:41:23.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:23 smithi019 ceph-mon[29267]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:41:23.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:23 smithi019 ceph-mon[29267]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:41:23.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:23 smithi019 ceph-mon[24574]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:41:23.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:23 smithi019 ceph-mon[24574]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:41:23.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:23 smithi172 ceph-mon[25457]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:41:23.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:23 smithi172 ceph-mon[25457]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:41:25.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:25 smithi019 ceph-mon[29267]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:41:25.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:25 smithi019 ceph-mon[24574]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:41:25.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:25 smithi172 ceph-mon[25457]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:41:27.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:27 smithi019 ceph-mon[29267]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:41:27.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:27 smithi019 ceph-mon[24574]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:41:27.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:27 smithi172 ceph-mon[25457]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:41:27.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:41:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:41:27] "GET /metrics HTTP/1.1" 200 37986 "" "Prometheus/2.33.4" 2024-02-07T12:41:29.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:29 smithi019 ceph-mon[29267]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:41:29.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:29 smithi019 ceph-mon[24574]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:41:29.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:29 smithi172 ceph-mon[25457]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:41:31.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:31 smithi019 ceph-mon[29267]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 554 B/s rd, 0 op/s 2024-02-07T12:41:31.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:31 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:41:31.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:31 smithi019 ceph-mon[24574]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 554 B/s rd, 0 op/s 2024-02-07T12:41:31.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:31 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:41:31.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:31 smithi172 ceph-mon[25457]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 554 B/s rd, 0 op/s 2024-02-07T12:41:31.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:31 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:41:33.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:33 smithi019 ceph-mon[29267]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:41:33.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:33 smithi019 ceph-mon[24574]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:41:33.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:33 smithi172 ceph-mon[25457]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:41:34.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:34 smithi019 ceph-mon[29267]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:34.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:34 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:41:34.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:34 smithi019 ceph-mon[24574]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:34.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:34 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:41:34.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:34 smithi172 ceph-mon[25457]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:34.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:34 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:41:37.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:36 smithi019 ceph-mon[24574]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:41:37.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:36 smithi019 ceph-mon[29267]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:41:37.276 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:36 smithi172 ceph-mon[25457]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:41:37.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:41:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:41:37] "GET /metrics HTTP/1.1" 200 37984 "" "Prometheus/2.33.4" 2024-02-07T12:41:39.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:39 smithi019 ceph-mon[29267]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:39.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:39 smithi019 ceph-mon[24574]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:39.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:39 smithi172 ceph-mon[25457]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:41.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:41 smithi019 ceph-mon[29267]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:41.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:41 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:41:41.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:41 smithi019 ceph-mon[24574]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:41.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:41 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:41:41.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:41 smithi172 ceph-mon[25457]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:41.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:41 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:41:43.437 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:43 smithi172 ceph-mon[25457]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:41:43.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:43 smithi019 ceph-mon[24574]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:41:43.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:43 smithi019 ceph-mon[29267]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:41:45.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:45 smithi019 ceph-mon[29267]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:45.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:45 smithi019 ceph-mon[24574]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:45.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:45 smithi172 ceph-mon[25457]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:47.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:47 smithi019 ceph-mon[24574]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:41:47.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:47 smithi019 ceph-mon[29267]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:41:47.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:47 smithi172 ceph-mon[25457]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:41:47.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:41:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:41:47] "GET /metrics HTTP/1.1" 200 37984 "" "Prometheus/2.33.4" 2024-02-07T12:41:49.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:49 smithi019 ceph-mon[24574]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:49.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:49 smithi019 ceph-mon[29267]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:49.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:49 smithi172 ceph-mon[25457]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:50.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:50 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:41:50.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:50 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:41:50.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:50 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:41:51.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:51 smithi019 ceph-mon[29267]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:51.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:51 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:41:51.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:51 smithi019 ceph-mon[24574]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:51.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:51 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:41:51.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:51 smithi172 ceph-mon[25457]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:51.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:51 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:41:53.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:53 smithi019 ceph-mon[29267]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:41:53.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:53 smithi019 ceph-mon[24574]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:41:53.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:53 smithi172 ceph-mon[25457]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:41:55.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:55 smithi019 ceph-mon[24574]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:55.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:55 smithi019 ceph-mon[29267]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:55.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:55 smithi172 ceph-mon[25457]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:57.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:57 smithi019 ceph-mon[24574]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:41:57.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:57 smithi019 ceph-mon[29267]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:41:57.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:57 smithi172 ceph-mon[25457]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:41:57.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:41:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:41:57] "GET /metrics HTTP/1.1" 200 37982 "" "Prometheus/2.33.4" 2024-02-07T12:41:59.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:41:59 smithi019 ceph-mon[24574]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:59.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:41:59 smithi019 ceph-mon[29267]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:41:59.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:41:59 smithi172 ceph-mon[25457]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:01.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:01 smithi019 ceph-mon[29267]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:01.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:01 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:42:01.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:01 smithi019 ceph-mon[24574]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:01.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:01 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:42:01.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:01 smithi172 ceph-mon[25457]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:01.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:01 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:42:03.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:03 smithi019 ceph-mon[24574]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:03.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:03 smithi019 ceph-mon[29267]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:03.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:03 smithi172 ceph-mon[25457]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:05.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:05 smithi019 ceph-mon[29267]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:05.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:05 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:42:05.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:05 smithi019 ceph-mon[24574]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:05.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:05 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:42:05.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:05 smithi172 ceph-mon[25457]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:05.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:05 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:42:07.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:07 smithi019 ceph-mon[29267]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:07.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:07 smithi019 ceph-mon[24574]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:07.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:07 smithi172 ceph-mon[25457]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:07.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:42:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:42:07] "GET /metrics HTTP/1.1" 200 37984 "" "Prometheus/2.33.4" 2024-02-07T12:42:09.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:09 smithi019 ceph-mon[29267]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:09.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:09 smithi019 ceph-mon[24574]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:09.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:09 smithi172 ceph-mon[25457]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:11.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:11 smithi019 ceph-mon[29267]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:11.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:11 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:42:11.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:11 smithi019 ceph-mon[24574]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:11 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:42:11.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:11 smithi172 ceph-mon[25457]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:11.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:11 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:42:13.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:13 smithi019 ceph-mon[29267]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:13.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:13 smithi019 ceph-mon[24574]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:13.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:13 smithi172 ceph-mon[25457]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:15.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:15 smithi019 ceph-mon[24574]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:15.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:15 smithi019 ceph-mon[29267]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:15.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:15 smithi172 ceph-mon[25457]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:17.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:17 smithi019 ceph-mon[24574]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:17.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:17 smithi019 ceph-mon[29267]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:17.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:17 smithi172 ceph-mon[25457]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:17.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:42:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:42:17] "GET /metrics HTTP/1.1" 200 37984 "" "Prometheus/2.33.4" 2024-02-07T12:42:19.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:19 smithi019 ceph-mon[24574]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:19.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:19 smithi019 ceph-mon[29267]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:19.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:19 smithi172 ceph-mon[25457]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:20.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:20 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:42:20.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:20 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:42:20.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:20 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:42:21.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:21 smithi019 ceph-mon[29267]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:21.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:21 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:42:21.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:21 smithi019 ceph-mon[24574]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:21.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:21 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:42:21.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:21 smithi172 ceph-mon[25457]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:21.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:21 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:42:22.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:22 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:42:22.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:22 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:42:22.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:22 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:42:23.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:23 smithi019 ceph-mon[29267]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:23.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:23 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:42:23.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:23 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:42:23.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:23 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:42:23.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:23 smithi019 ceph-mon[24574]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:23.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:23 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:42:23.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:23 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:42:23.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:23 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:42:23.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:23 smithi172 ceph-mon[25457]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:23.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:23 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:42:23.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:23 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:42:23.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:23 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:42:24.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:24 smithi019 ceph-mon[29267]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 955 B/s rd, 0 op/s 2024-02-07T12:42:24.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:24 smithi019 ceph-mon[29267]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:42:24.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:24 smithi019 ceph-mon[24574]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 955 B/s rd, 0 op/s 2024-02-07T12:42:24.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:24 smithi019 ceph-mon[24574]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:42:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:24 smithi172 ceph-mon[25457]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 955 B/s rd, 0 op/s 2024-02-07T12:42:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:24 smithi172 ceph-mon[25457]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:42:26.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:26 smithi019 ceph-mon[24574]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 587 B/s rd, 0 op/s 2024-02-07T12:42:26.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:26 smithi019 ceph-mon[29267]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 587 B/s rd, 0 op/s 2024-02-07T12:42:26.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:26 smithi172 ceph-mon[25457]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 587 B/s rd, 0 op/s 2024-02-07T12:42:27.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:42:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:42:27] "GET /metrics HTTP/1.1" 200 37978 "" "Prometheus/2.33.4" 2024-02-07T12:42:28.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:28 smithi019 ceph-mon[24574]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:42:28.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:28 smithi019 ceph-mon[29267]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:42:28.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:28 smithi172 ceph-mon[25457]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:42:30.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:30 smithi019 ceph-mon[24574]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:42:30.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:30 smithi019 ceph-mon[29267]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:42:30.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:30 smithi172 ceph-mon[25457]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:42:31.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:31 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:42:31.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:31 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:42:31.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:31 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:42:32.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:32 smithi019 ceph-mon[24574]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:42:32.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:32 smithi019 ceph-mon[29267]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:42:32.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:32 smithi172 ceph-mon[25457]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:42:34.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:34 smithi019 ceph-mon[24574]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:42:34.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:34 smithi019 ceph-mon[29267]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:42:34.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:34 smithi172 ceph-mon[25457]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:42:35.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:35 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:42:35.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:35 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:42:35.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:35 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:42:36.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:36 smithi019 ceph-mon[24574]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:36.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:36 smithi019 ceph-mon[29267]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:36.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:36 smithi172 ceph-mon[25457]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:37.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:42:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:42:37] "GET /metrics HTTP/1.1" 200 37975 "" "Prometheus/2.33.4" 2024-02-07T12:42:38.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:38 smithi019 ceph-mon[24574]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:38.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:38 smithi019 ceph-mon[29267]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:38.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:38 smithi172 ceph-mon[25457]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:40.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:40 smithi019 ceph-mon[29267]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:40.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:40 smithi019 ceph-mon[24574]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:40.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:40 smithi172 ceph-mon[25457]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:41.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:41 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:42:41.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:41 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:42:41.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:41 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:42:42.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:42 smithi019 ceph-mon[29267]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:42.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:42 smithi019 ceph-mon[24574]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:42.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:42 smithi172 ceph-mon[25457]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:44.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:44 smithi019 ceph-mon[24574]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:44.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:44 smithi019 ceph-mon[29267]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:44.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:44 smithi172 ceph-mon[25457]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:46.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:46 smithi019 ceph-mon[24574]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:46.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:46 smithi019 ceph-mon[29267]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:46.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:46 smithi172 ceph-mon[25457]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:47.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:42:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:42:47] "GET /metrics HTTP/1.1" 200 37975 "" "Prometheus/2.33.4" 2024-02-07T12:42:48.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:48 smithi019 ceph-mon[24574]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:48.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:48 smithi019 ceph-mon[29267]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:48.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:48 smithi172 ceph-mon[25457]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:49.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:49 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:42:49.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:49 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:42:49.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:49 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:42:50.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:50 smithi019 ceph-mon[24574]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:50.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:50 smithi019 ceph-mon[29267]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:50.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:50 smithi172 ceph-mon[25457]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:51.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:51 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:42:51.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:51 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:42:51.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:51 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:42:52.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:52 smithi019 ceph-mon[24574]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:52.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:52 smithi019 ceph-mon[29267]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:52.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:52 smithi172 ceph-mon[25457]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:54.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:54 smithi019 ceph-mon[29267]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:54.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:54 smithi019 ceph-mon[24574]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:54.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:54 smithi172 ceph-mon[25457]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:56.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:56 smithi019 ceph-mon[24574]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:56.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:56 smithi019 ceph-mon[29267]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:56.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:56 smithi172 ceph-mon[25457]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:42:57.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:42:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:42:57] "GET /metrics HTTP/1.1" 200 37981 "" "Prometheus/2.33.4" 2024-02-07T12:42:58.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:42:58 smithi019 ceph-mon[24574]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:58.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:42:58 smithi019 ceph-mon[29267]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:42:58.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:42:58 smithi172 ceph-mon[25457]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:00.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:00 smithi019 ceph-mon[24574]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:00.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:00 smithi019 ceph-mon[29267]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:00.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:00 smithi172 ceph-mon[25457]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:01.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:01 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:43:01.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:01 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:43:01.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:01 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:43:02.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:02 smithi019 ceph-mon[29267]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:02.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:02 smithi019 ceph-mon[24574]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:02.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:02 smithi172 ceph-mon[25457]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:04.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:04 smithi019 ceph-mon[29267]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:04.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:04 smithi019 ceph-mon[24574]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:04.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:04 smithi172 ceph-mon[25457]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:05.495 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:05 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:43:05.496 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:05 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:43:05.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:05 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:43:06.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:06 smithi019 ceph-mon[29267]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:06.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:06 smithi019 ceph-mon[24574]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:06.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:06 smithi172 ceph-mon[25457]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:07.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:43:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:43:07] "GET /metrics HTTP/1.1" 200 37980 "" "Prometheus/2.33.4" 2024-02-07T12:43:08.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:08 smithi019 ceph-mon[24574]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:08.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:08 smithi019 ceph-mon[29267]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:08.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:08 smithi172 ceph-mon[25457]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:10.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:10 smithi019 ceph-mon[29267]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:10.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:10 smithi019 ceph-mon[24574]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:10.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:10 smithi172 ceph-mon[25457]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:11.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:11 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:43:11.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:11 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:43:11.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:11 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:43:12.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:12 smithi019 ceph-mon[29267]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:12.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:12 smithi019 ceph-mon[24574]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:12.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:12 smithi172 ceph-mon[25457]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:14.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:14 smithi019 ceph-mon[29267]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:14.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:14 smithi019 ceph-mon[24574]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:14.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:14 smithi172 ceph-mon[25457]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:16.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:16 smithi019 ceph-mon[24574]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:16.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:16 smithi019 ceph-mon[29267]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:16.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:16 smithi172 ceph-mon[25457]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:17.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:43:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:43:17] "GET /metrics HTTP/1.1" 200 37980 "" "Prometheus/2.33.4" 2024-02-07T12:43:18.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:18 smithi019 ceph-mon[29267]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:18.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:18 smithi019 ceph-mon[24574]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:18.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:18 smithi172 ceph-mon[25457]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:20.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:20 smithi019 ceph-mon[24574]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:20.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:20 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:43:20.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:20 smithi019 ceph-mon[29267]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:20.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:20 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:43:20.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:20 smithi172 ceph-mon[25457]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:20.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:20 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:43:21.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:21 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:43:21.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:21 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:43:21.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:21 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:43:22.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:22 smithi019 ceph-mon[24574]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:22.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:22 smithi019 ceph-mon[29267]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:22.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:22 smithi172 ceph-mon[25457]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:23.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:23 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:43:23.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:23 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:43:23.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:23 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:43:24.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:24 smithi019 ceph-mon[24574]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:24.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:24 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:43:24.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:24 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:43:24.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:24 smithi019 ceph-mon[24574]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 958 B/s rd, 0 op/s 2024-02-07T12:43:24.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:24 smithi019 ceph-mon[24574]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:24.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:24 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:43:24.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:24 smithi019 ceph-mon[29267]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:24.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:24 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:43:24.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:24 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:43:24.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:24 smithi019 ceph-mon[29267]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 958 B/s rd, 0 op/s 2024-02-07T12:43:24.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:24 smithi019 ceph-mon[29267]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:24.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:24 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:43:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:24 smithi172 ceph-mon[25457]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:24 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:43:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:24 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:43:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:24 smithi172 ceph-mon[25457]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 958 B/s rd, 0 op/s 2024-02-07T12:43:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:24 smithi172 ceph-mon[25457]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:24 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:43:26.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:26 smithi019 ceph-mon[24574]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:26.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:26 smithi019 ceph-mon[29267]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:26.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:26 smithi172 ceph-mon[25457]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:27.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:43:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:43:27] "GET /metrics HTTP/1.1" 200 37991 "" "Prometheus/2.33.4" 2024-02-07T12:43:28.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:28 smithi019 ceph-mon[24574]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:28.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:28 smithi019 ceph-mon[29267]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:28.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:28 smithi172 ceph-mon[25457]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:30.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:30 smithi019 ceph-mon[29267]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 589 B/s rd, 0 op/s 2024-02-07T12:43:30.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:30 smithi019 ceph-mon[24574]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 589 B/s rd, 0 op/s 2024-02-07T12:43:30.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:30 smithi172 ceph-mon[25457]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 589 B/s rd, 0 op/s 2024-02-07T12:43:31.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:31 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:43:31.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:31 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:43:31.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:31 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:43:32.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:32 smithi019 ceph-mon[24574]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:32.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:32 smithi019 ceph-mon[29267]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:32.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:32 smithi172 ceph-mon[25457]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:34.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:34 smithi019 ceph-mon[24574]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:43:34.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:34 smithi019 ceph-mon[29267]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:43:34.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:34 smithi172 ceph-mon[25457]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:43:35.495 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:35 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:43:35.496 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:35 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:43:35.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:35 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:43:36.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:36 smithi019 ceph-mon[24574]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:36.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:36 smithi019 ceph-mon[29267]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:36.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:36 smithi172 ceph-mon[25457]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:37.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:43:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:43:37] "GET /metrics HTTP/1.1" 200 37984 "" "Prometheus/2.33.4" 2024-02-07T12:43:38.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:38 smithi019 ceph-mon[24574]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-02-07T12:43:38.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:38 smithi019 ceph-mon[29267]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-02-07T12:43:38.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:38 smithi172 ceph-mon[25457]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-02-07T12:43:40.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:40 smithi019 ceph-mon[24574]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-02-07T12:43:40.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:40 smithi019 ceph-mon[29267]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-02-07T12:43:40.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:40 smithi172 ceph-mon[25457]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-02-07T12:43:41.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:41 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:43:41.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:41 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:43:41.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:41 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:43:42.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:42 smithi019 ceph-mon[29267]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:42.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:42 smithi019 ceph-mon[24574]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:42.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:42 smithi172 ceph-mon[25457]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:44.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:44 smithi019 ceph-mon[29267]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-02-07T12:43:44.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:44 smithi019 ceph-mon[24574]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-02-07T12:43:44.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:44 smithi172 ceph-mon[25457]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-02-07T12:43:46.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:46 smithi019 ceph-mon[29267]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:46.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:46 smithi019 ceph-mon[24574]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:46.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:46 smithi172 ceph-mon[25457]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:47.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:43:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:43:47] "GET /metrics HTTP/1.1" 200 37984 "" "Prometheus/2.33.4" 2024-02-07T12:43:48.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:48 smithi019 ceph-mon[24574]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:48.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:48 smithi019 ceph-mon[29267]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:48.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:48 smithi172 ceph-mon[25457]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:49.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:49 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:43:49.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:49 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:43:49.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:49 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:43:50.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:50 smithi019 ceph-mon[24574]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:50.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:50 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:43:50.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:50 smithi019 ceph-mon[29267]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:50.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:50 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:43:50.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:50 smithi172 ceph-mon[25457]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:50.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:50 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:43:51.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:51 smithi019 ceph-mon[24574]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:51.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:51 smithi019 ceph-mon[29267]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:51.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:51 smithi172 ceph-mon[25457]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:54.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:54 smithi019 ceph-mon[24574]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:54.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:54 smithi019 ceph-mon[29267]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:54.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:54 smithi172 ceph-mon[25457]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:56.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:56 smithi019 ceph-mon[24574]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:56.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:56 smithi019 ceph-mon[29267]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:56.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:56 smithi172 ceph-mon[25457]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:43:57.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:43:57 smithi019 ceph-mon[24574]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:57.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:43:57 smithi019 ceph-mon[29267]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:43:57.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:43:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:43:57] "GET /metrics HTTP/1.1" 200 37988 "" "Prometheus/2.33.4" 2024-02-07T12:43:57.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:43:57 smithi172 ceph-mon[25457]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:44:00.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:00 smithi019 ceph-mon[29267]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:44:00.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:00 smithi019 ceph-mon[24574]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:44:00.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:00 smithi172 ceph-mon[25457]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:44:01.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:01 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:44:01.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:01 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:44:01.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:01 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:44:02.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:02 smithi019 ceph-mon[24574]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:44:02.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:02 smithi019 ceph-mon[29267]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:44:02.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:02 smithi172 ceph-mon[25457]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:44:04.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:04 smithi019 ceph-mon[24574]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:44:04.843 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:04 smithi019 ceph-mon[29267]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:44:04.844 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:04 smithi172 ceph-mon[25457]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:44:05.495 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:05 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:44:05.496 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:05 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:44:05.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:05 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:44:06.495 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:06 smithi019 ceph-mon[24574]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:44:06.495 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:06 smithi019 ceph-mon[29267]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:44:06.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:06 smithi172 ceph-mon[25457]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:44:07.131 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch ps' 2024-02-07T12:44:07.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:44:07] "GET /metrics HTTP/1.1" 200 37987 "" "Prometheus/2.33.4" 2024-02-07T12:44:08.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:08 smithi019 ceph-mon[29267]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:44:08.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:08 smithi019 ceph-mon[24574]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:44:08.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:08 smithi172 ceph-mon[25457]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:44:09.139 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:44:09.139 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (12m) 2m ago 20m 23.8M - 0.23.0 ba2b418f427c dfaf78b29ab5 2024-02-07T12:44:09.139 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (11m) 2m ago 20m 52.1M - 8.3.5 dad864ee21e9 e753e96fa4f0 2024-02-07T12:44:09.140 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 error 2m ago 19m 54.5M - 3.5 e1d6a67b021e 5113aa77e3f2 2024-02-07T12:44:09.140 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283 running (15m) 2m ago 24m 491M - 19.0.0-1252-g44458cce 79957b372307 97644267e529 2024-02-07T12:44:09.140 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (9m) 2m ago 26m 71.7M - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:44:09.140 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (26m) 2m ago 26m 68.3M 2048M 17.2.0 e1d6a67b021e f3a8a050cf1d 2024-02-07T12:44:09.140 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (25m) 2m ago 24m 52.1M 2048M 17.2.0 e1d6a67b021e 7d62c7c4d075 2024-02-07T12:44:09.140 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (25m) 2m ago 25m 52.1M 2048M 17.2.0 e1d6a67b021e b9825d0db32d 2024-02-07T12:44:09.140 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (12m) 2m ago 20m 21.4M - 1.3.1 1dbe0e931976 9868df14bcc9 2024-02-07T12:44:09.140 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (12m) 2m ago 20m 17.5M - 1.3.1 1dbe0e931976 e066f1871320 2024-02-07T12:44:09.140 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (24m) 2m ago 24m 53.3M 2993M 17.2.0 e1d6a67b021e 87f2560a9b15 2024-02-07T12:44:09.140 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (24m) 2m ago 24m 55.7M 2993M 17.2.0 e1d6a67b021e 0f9a190f0e0d 2024-02-07T12:44:09.140 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (23m) 2m ago 23m 52.3M 2993M 17.2.0 e1d6a67b021e 7db94f25a1aa 2024-02-07T12:44:09.140 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (23m) 2m ago 23m 52.9M 2993M 17.2.0 e1d6a67b021e 0b6b674188ed 2024-02-07T12:44:09.140 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (22m) 2m ago 22m 52.7M 3248M 17.2.0 e1d6a67b021e df72cfd69b3b 2024-02-07T12:44:09.140 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (22m) 2m ago 22m 51.8M 3248M 17.2.0 e1d6a67b021e 3bdb22c07537 2024-02-07T12:44:09.141 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (22m) 2m ago 22m 50.5M 3248M 17.2.0 e1d6a67b021e 7859b651e0d9 2024-02-07T12:44:09.141 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (21m) 2m ago 21m 54.6M 3248M 17.2.0 e1d6a67b021e b301e7c75d73 2024-02-07T12:44:09.141 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (2m) 2m ago 21m 46.8M - 2.33.4 514e6a882f6e 6b7568bd4b22 2024-02-07T12:44:09.141 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (19m) 2m ago 19m 76.6M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:44:09.141 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (19m) 2m ago 19m 76.2M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:44:09.724 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch ls' 2024-02-07T12:44:10.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:10 smithi019 ceph-mon[24574]: from='client.25204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:44:10.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:10 smithi019 ceph-mon[24574]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:44:10.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:10 smithi019 ceph-mon[29267]: from='client.25204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:44:10.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:10 smithi019 ceph-mon[29267]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:44:10.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:10 smithi172 ceph-mon[25457]: from='client.25204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:44:10.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:10 smithi172 ceph-mon[25457]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:44:11.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:11 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:44:11.586 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:11 smithi019 ceph-mon[24574]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:44:11.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:11 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:44:11.586 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:11 smithi019 ceph-mon[29267]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:44:11.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:11 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:44:11.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:11 smithi172 ceph-mon[25457]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:44:11.789 INFO:teuthology.orchestra.run.smithi019.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-02-07T12:44:11.790 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager ?:9093,9094 1/1 3m ago 21m smithi019=a;count:1 2024-02-07T12:44:11.790 INFO:teuthology.orchestra.run.smithi019.stdout:grafana ?:3000 1/1 2m ago 21m smithi172=a;count:1 2024-02-07T12:44:11.790 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo ?:5000 0/1 3m ago 19m count:1 2024-02-07T12:44:11.790 INFO:teuthology.orchestra.run.smithi019.stdout:mgr 2/2 3m ago 24m smithi019=y;smithi172=x;count:2 2024-02-07T12:44:11.790 INFO:teuthology.orchestra.run.smithi019.stdout:mon 3/3 3m ago 25m smithi019:172.21.15.19=a;smithi019:[v2:172.21.15.19:3301,v1:172.21.15.19:6790]=c;smithi172:172.21.15.172=b;count:3 2024-02-07T12:44:11.790 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter ?:9100 2/2 3m ago 21m smithi019=a;smithi172=b;count:2 2024-02-07T12:44:11.790 INFO:teuthology.orchestra.run.smithi019.stdout:osd 8 3m ago - 2024-02-07T12:44:11.790 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus ?:9095 1/1 2m ago 21m smithi172=a;count:1 2024-02-07T12:44:11.790 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo ?:8000 2/2 3m ago 19m count:2 2024-02-07T12:44:12.532 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:12 smithi019 ceph-mon[29267]: from='client.15324 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:44:12.532 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:12 smithi019 ceph-mon[24574]: from='client.15324 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:44:12.559 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph versions' 2024-02-07T12:44:12.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:12 smithi172 ceph-mon[25457]: from='client.15324 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:44:14.342 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:14 smithi019 ceph-mon[29267]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:44:14.342 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:14 smithi019 ceph-mon[24574]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:44:14.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:14 smithi172 ceph-mon[25457]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:44:14.684 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:44:14.684 INFO:teuthology.orchestra.run.smithi019.stdout: "mon": { 2024-02-07T12:44:14.684 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-02-07T12:44:14.685 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:44:14.685 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr": { 2024-02-07T12:44:14.685 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:44:14.685 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:44:14.685 INFO:teuthology.orchestra.run.smithi019.stdout: "osd": { 2024-02-07T12:44:14.685 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-02-07T12:44:14.685 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:44:14.685 INFO:teuthology.orchestra.run.smithi019.stdout: "mds": {}, 2024-02-07T12:44:14.685 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw": { 2024-02-07T12:44:14.685 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:44:14.685 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:44:14.685 INFO:teuthology.orchestra.run.smithi019.stdout: "overall": { 2024-02-07T12:44:14.686 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-02-07T12:44:14.686 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:44:14.686 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:44:14.686 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:44:15.288 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:15 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1536622524' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:15.288 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:15 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1536622524' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:15.311 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr' 2024-02-07T12:44:15.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:15 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1536622524' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:16.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:16 smithi019 ceph-mon[29267]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:44:16.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:16 smithi019 ceph-mon[24574]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:44:16.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:16 smithi172 ceph-mon[25457]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:44:17.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:44:17] "GET /metrics HTTP/1.1" 200 37987 "" "Prometheus/2.33.4" 2024-02-07T12:44:18.593 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:18 smithi019 ceph-mon[29267]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:44:18.593 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:18 smithi019 ceph-mon[24574]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:44:18.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:18 smithi172 ceph-mon[25457]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:44:19.586 INFO:teuthology.orchestra.run.smithi019.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:44:19.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:19 smithi019 ceph-mon[29267]: from='client.25222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:44:19.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:19 smithi019 ceph-mon[29267]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:44:19.587 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:19 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:44:19.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:19 smithi019 ceph-mon[24574]: from='client.25222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:44:19.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:19 smithi019 ceph-mon[24574]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:44:19.587 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:19 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:44:19.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:19 smithi172 ceph-mon[25457]: from='client.25222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:44:19.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:19 smithi172 ceph-mon[25457]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:44:19.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:19 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:44:20.159 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-02-07T12:44:20.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:20 smithi019 ceph-mon[29267]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:44:20.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:20 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:44:20.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:20 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:44:20.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:20 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:44:20.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:20 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:44:20.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:20 smithi019 ceph-mon[29267]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 987 B/s rd, 0 op/s 2024-02-07T12:44:20.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:20 smithi019 ceph-mon[29267]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 611 B/s rd, 0 op/s 2024-02-07T12:44:20.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:20 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:44:20.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:20 smithi019 ceph-mon[29267]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:44:20.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:20 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:44:20.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:20 smithi019 ceph-mon[24574]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:44:20.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:20 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:44:20.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:20 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:44:20.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:20 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:44:20.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:20 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:44:20.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:20 smithi019 ceph-mon[24574]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 987 B/s rd, 0 op/s 2024-02-07T12:44:20.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:20 smithi019 ceph-mon[24574]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 611 B/s rd, 0 op/s 2024-02-07T12:44:20.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:20 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:44:20.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:20 smithi019 ceph-mon[24574]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:44:20.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:20 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:44:20.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:20 smithi172 ceph-mon[25457]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:44:20.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:20 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:44:20.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:20 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:44:20.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:20 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:44:20.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:20 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:44:20.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:20 smithi172 ceph-mon[25457]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 987 B/s rd, 0 op/s 2024-02-07T12:44:20.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:20 smithi172 ceph-mon[25457]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 611 B/s rd, 0 op/s 2024-02-07T12:44:20.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:20 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:44:20.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:20 smithi172 ceph-mon[25457]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:44:20.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:20 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:44:23.033 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:22 smithi019 ceph-mon[24574]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:44:23.033 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:22 smithi019 ceph-mon[29267]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:44:23.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:22 smithi172 ceph-mon[25457]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:44:23.402 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:44:24.211 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:23 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:44:24.212 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:23 smithi172 ceph-mon[25457]: Upgrade: Target is version 19.0.0-1252-g44458cce (squid) 2024-02-07T12:44:24.212 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:23 smithi172 ceph-mon[25457]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe'] 2024-02-07T12:44:24.212 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:23 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:44:24.212 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:23 smithi172 ceph-mon[25457]: Upgrade: Need to upgrade myself (mgr.x) 2024-02-07T12:44:24.212 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:23 smithi172 ceph-mon[25457]: Upgrade: Need to upgrade myself (mgr.x) 2024-02-07T12:44:24.212 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:23 smithi172 ceph-mon[25457]: Failing over to other MGR 2024-02-07T12:44:24.212 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:23 smithi172 ceph-mon[25457]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-02-07T12:44:24.212 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:23 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-02-07T12:44:24.212 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:23 smithi172 ceph-mon[25457]: osdmap e104: 8 total, 8 up, 8 in 2024-02-07T12:44:24.212 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:23 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:44:23.959+0000 7f0601ac1700 -1 mgr handle_mgr_map I was active but no longer am 2024-02-07T12:44:24.213 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ignoring --setuser ceph since I am not root 2024-02-07T12:44:24.213 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: ignoring --setgroup ceph since I am not root 2024-02-07T12:44:24.213 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-mgr[45648]: -- 172.21.15.172:0/533407194 <== mon.2 v2:172.21.15.172:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559c53479520 con 0x559c5344f000 2024-02-07T12:44:24.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: [07/Feb/2024:12:44:23] ENGINE Bus STOPPING 2024-02-07T12:44:24.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: [07/Feb/2024:12:44:23] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-02-07T12:44:24.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: [07/Feb/2024:12:44:23] ENGINE Bus STOPPED 2024-02-07T12:44:24.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: [07/Feb/2024:12:44:24] ENGINE Bus STARTING 2024-02-07T12:44:24.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: [07/Feb/2024:12:44:24] ENGINE Serving on http://:::9283 2024-02-07T12:44:24.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: [07/Feb/2024:12:44:24] ENGINE Bus STARTED 2024-02-07T12:44:24.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:23 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:44:24.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:23 smithi019 ceph-mon[29267]: Upgrade: Target is version 19.0.0-1252-g44458cce (squid) 2024-02-07T12:44:24.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:23 smithi019 ceph-mon[29267]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe'] 2024-02-07T12:44:24.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:23 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:44:24.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:23 smithi019 ceph-mon[29267]: Upgrade: Need to upgrade myself (mgr.x) 2024-02-07T12:44:24.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:23 smithi019 ceph-mon[29267]: Upgrade: Need to upgrade myself (mgr.x) 2024-02-07T12:44:24.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:23 smithi019 ceph-mon[29267]: Failing over to other MGR 2024-02-07T12:44:24.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:23 smithi019 ceph-mon[29267]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-02-07T12:44:24.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:23 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-02-07T12:44:24.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:23 smithi019 ceph-mon[29267]: osdmap e104: 8 total, 8 up, 8 in 2024-02-07T12:44:24.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:23 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' 2024-02-07T12:44:24.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:23 smithi019 ceph-mon[24574]: Upgrade: Target is version 19.0.0-1252-g44458cce (squid) 2024-02-07T12:44:24.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:23 smithi019 ceph-mon[24574]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe'] 2024-02-07T12:44:24.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:23 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:44:24.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:23 smithi019 ceph-mon[24574]: Upgrade: Need to upgrade myself (mgr.x) 2024-02-07T12:44:24.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:23 smithi019 ceph-mon[24574]: Upgrade: Need to upgrade myself (mgr.x) 2024-02-07T12:44:24.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:23 smithi019 ceph-mon[24574]: Failing over to other MGR 2024-02-07T12:44:24.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:23 smithi019 ceph-mon[24574]: from='mgr.24944 172.21.15.172:0/2893558188' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-02-07T12:44:24.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:23 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-02-07T12:44:24.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:23 smithi019 ceph-mon[24574]: osdmap e104: 8 total, 8 up, 8 in 2024-02-07T12:44:24.869 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:44:24.540+0000 7f05e89cc200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-02-07T12:44:24.869 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:44:24.637+0000 7f05e89cc200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-02-07T12:44:24.990 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:44:24.990 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (12m) 3m ago 20m 23.8M - 0.23.0 ba2b418f427c dfaf78b29ab5 2024-02-07T12:44:24.990 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (12m) 3m ago 20m 52.1M - 8.3.5 dad864ee21e9 e753e96fa4f0 2024-02-07T12:44:24.990 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 error 3m ago 19m 54.5M - 3.5 e1d6a67b021e 5113aa77e3f2 2024-02-07T12:44:24.990 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283 running (15m) 3m ago 25m 491M - 19.0.0-1252-g44458cce 79957b372307 97644267e529 2024-02-07T12:44:24.990 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (9m) 3m ago 26m 71.7M - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:44:24.990 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (26m) 3m ago 26m 68.3M 2048M 17.2.0 e1d6a67b021e f3a8a050cf1d 2024-02-07T12:44:24.991 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (25m) 3m ago 25m 52.1M 2048M 17.2.0 e1d6a67b021e 7d62c7c4d075 2024-02-07T12:44:24.991 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (25m) 3m ago 25m 52.1M 2048M 17.2.0 e1d6a67b021e b9825d0db32d 2024-02-07T12:44:24.991 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (12m) 3m ago 21m 21.4M - 1.3.1 1dbe0e931976 9868df14bcc9 2024-02-07T12:44:24.991 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (12m) 3m ago 21m 17.5M - 1.3.1 1dbe0e931976 e066f1871320 2024-02-07T12:44:24.991 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (24m) 3m ago 24m 53.3M 2993M 17.2.0 e1d6a67b021e 87f2560a9b15 2024-02-07T12:44:24.991 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (24m) 3m ago 24m 55.7M 2993M 17.2.0 e1d6a67b021e 0f9a190f0e0d 2024-02-07T12:44:24.991 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (23m) 3m ago 23m 52.3M 2993M 17.2.0 e1d6a67b021e 7db94f25a1aa 2024-02-07T12:44:24.991 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (23m) 3m ago 23m 52.9M 2993M 17.2.0 e1d6a67b021e 0b6b674188ed 2024-02-07T12:44:24.991 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (23m) 3m ago 23m 52.7M 3248M 17.2.0 e1d6a67b021e df72cfd69b3b 2024-02-07T12:44:24.991 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (22m) 3m ago 22m 51.8M 3248M 17.2.0 e1d6a67b021e 3bdb22c07537 2024-02-07T12:44:24.991 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (22m) 3m ago 22m 50.5M 3248M 17.2.0 e1d6a67b021e 7859b651e0d9 2024-02-07T12:44:24.991 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (22m) 3m ago 22m 54.6M 3248M 17.2.0 e1d6a67b021e b301e7c75d73 2024-02-07T12:44:24.991 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (3m) 3m ago 21m 46.8M - 2.33.4 514e6a882f6e 6b7568bd4b22 2024-02-07T12:44:24.991 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (19m) 3m ago 19m 76.6M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:44:24.991 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (19m) 3m ago 19m 76.2M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:44:25.220 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[29267]: from='mgr.24944 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-02-07T12:44:25.220 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[29267]: mgrmap e39: y(active, starting, since 0.99533s) 2024-02-07T12:44:25.220 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:44:25.220 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:44:25.221 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:44:25.221 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-02-07T12:44:25.221 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:44:25.221 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:44:25.221 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:44:25.221 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:44:25.221 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:44:25.221 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:44:25.221 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:44:25.221 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:44:25.221 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:44:25.221 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:44:25.221 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:44:25.221 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[29267]: Manager daemon y is now available 2024-02-07T12:44:25.221 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:44:25.222 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:44:25.222 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:44:25.222 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:44:25.222 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:44:25.222 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:44:25.222 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:44:25.222 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[24574]: from='mgr.24944 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-02-07T12:44:25.222 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[24574]: mgrmap e39: y(active, starting, since 0.99533s) 2024-02-07T12:44:25.222 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:44:25.222 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:44:25.222 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:44:25.222 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-02-07T12:44:25.222 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:44:25.223 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:44:25.223 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:44:25.223 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:44:25.223 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:44:25.223 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:44:25.223 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:44:25.223 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:44:25.223 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:44:25.223 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:44:25.223 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:44:25.223 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[24574]: Manager daemon y is now available 2024-02-07T12:44:25.223 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:44:25.223 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:44:25.223 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:44:25.223 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:44:25.224 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:44:25.224 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:44:25.224 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:24 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:44:25.257 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-mon[25457]: from='mgr.24944 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-02-07T12:44:25.257 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-mon[25457]: mgrmap e39: y(active, starting, since 0.99533s) 2024-02-07T12:44:25.257 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:44:25.257 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:44:25.257 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:44:25.257 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-02-07T12:44:25.257 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:44:25.257 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:44:25.257 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:44:25.257 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:44:25.258 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:44:25.258 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:44:25.258 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:44:25.258 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:44:25.258 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:44:25.258 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:44:25.258 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:44:25.258 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-mon[25457]: Manager daemon y is now available 2024-02-07T12:44:25.258 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:44:25.258 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:44:25.258 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:44:25.258 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:44:25.258 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:44:25.258 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:44:25.258 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:24 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:44:25.392 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:44:25.393 INFO:teuthology.orchestra.run.smithi019.stdout: "mon": { 2024-02-07T12:44:25.393 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-02-07T12:44:25.393 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:44:25.393 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr": { 2024-02-07T12:44:25.393 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 1 2024-02-07T12:44:25.393 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:44:25.393 INFO:teuthology.orchestra.run.smithi019.stdout: "osd": { 2024-02-07T12:44:25.393 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-02-07T12:44:25.393 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:44:25.393 INFO:teuthology.orchestra.run.smithi019.stdout: "mds": {}, 2024-02-07T12:44:25.393 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw": { 2024-02-07T12:44:25.393 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:44:25.393 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:44:25.394 INFO:teuthology.orchestra.run.smithi019.stdout: "overall": { 2024-02-07T12:44:25.394 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-02-07T12:44:25.394 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 1 2024-02-07T12:44:25.394 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:44:25.394 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:44:25.564 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:44:25.323+0000 7f05e89cc200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-02-07T12:44:25.565 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:44:25.415+0000 7f05e89cc200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-02-07T12:44:25.565 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:44:25.562+0000 7f05e89cc200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-02-07T12:44:25.739 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:44:25.740 INFO:teuthology.orchestra.run.smithi019.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:44:25.740 INFO:teuthology.orchestra.run.smithi019.stdout: "in_progress": true, 2024-02-07T12:44:25.740 INFO:teuthology.orchestra.run.smithi019.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-02-07T12:44:25.740 INFO:teuthology.orchestra.run.smithi019.stdout: "services_complete": [ 2024-02-07T12:44:25.740 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr" 2024-02-07T12:44:25.740 INFO:teuthology.orchestra.run.smithi019.stdout: ], 2024-02-07T12:44:25.740 INFO:teuthology.orchestra.run.smithi019.stdout: "progress": "2/2 daemons upgraded", 2024-02-07T12:44:25.740 INFO:teuthology.orchestra.run.smithi019.stdout: "message": "", 2024-02-07T12:44:25.740 INFO:teuthology.orchestra.run.smithi019.stdout: "is_paused": false 2024-02-07T12:44:25.740 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:44:25.836 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:44:25.674+0000 7f05e89cc200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-02-07T12:44:25.836 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:44:25.761+0000 7f05e89cc200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-02-07T12:44:26.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:25 smithi019 ceph-mon[29267]: [07/Feb/2024:12:44:24] ENGINE Bus STARTING 2024-02-07T12:44:26.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:25 smithi019 ceph-mon[29267]: [07/Feb/2024:12:44:24] ENGINE Serving on https://172.21.15.19:7150 2024-02-07T12:44:26.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:25 smithi019 ceph-mon[29267]: [07/Feb/2024:12:44:24] ENGINE Serving on http://172.21.15.19:8765 2024-02-07T12:44:26.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:25 smithi019 ceph-mon[29267]: [07/Feb/2024:12:44:24] ENGINE Bus STARTED 2024-02-07T12:44:26.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:25 smithi019 ceph-mon[29267]: mgrmap e40: y(active, since 2s) 2024-02-07T12:44:26.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:25 smithi019 ceph-mon[29267]: from='client.25240 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:44:26.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:25 smithi019 ceph-mon[29267]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:44:26.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:25 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/965694275' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:26.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:25 smithi019 ceph-mon[24574]: [07/Feb/2024:12:44:24] ENGINE Bus STARTING 2024-02-07T12:44:26.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:25 smithi019 ceph-mon[24574]: [07/Feb/2024:12:44:24] ENGINE Serving on https://172.21.15.19:7150 2024-02-07T12:44:26.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:25 smithi019 ceph-mon[24574]: [07/Feb/2024:12:44:24] ENGINE Serving on http://172.21.15.19:8765 2024-02-07T12:44:26.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:25 smithi019 ceph-mon[24574]: [07/Feb/2024:12:44:24] ENGINE Bus STARTED 2024-02-07T12:44:26.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:25 smithi019 ceph-mon[24574]: mgrmap e40: y(active, since 2s) 2024-02-07T12:44:26.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:25 smithi019 ceph-mon[24574]: from='client.25240 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:44:26.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:25 smithi019 ceph-mon[24574]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:44:26.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:25 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/965694275' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:26.119 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:26 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:44:26.034+0000 7f05e89cc200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-02-07T12:44:26.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:25 smithi172 ceph-mon[25457]: [07/Feb/2024:12:44:24] ENGINE Bus STARTING 2024-02-07T12:44:26.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:25 smithi172 ceph-mon[25457]: [07/Feb/2024:12:44:24] ENGINE Serving on https://172.21.15.19:7150 2024-02-07T12:44:26.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:25 smithi172 ceph-mon[25457]: [07/Feb/2024:12:44:24] ENGINE Serving on http://172.21.15.19:8765 2024-02-07T12:44:26.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:25 smithi172 ceph-mon[25457]: [07/Feb/2024:12:44:24] ENGINE Bus STARTED 2024-02-07T12:44:26.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:25 smithi172 ceph-mon[25457]: mgrmap e40: y(active, since 2s) 2024-02-07T12:44:26.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:25 smithi172 ceph-mon[25457]: from='client.25240 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:44:26.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:25 smithi172 ceph-mon[25457]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:44:26.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:25 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/965694275' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:26.538 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:26 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:44:26.148+0000 7f05e89cc200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-02-07T12:44:26.538 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:26 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:44:26.235+0000 7f05e89cc200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-02-07T12:44:26.538 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:26 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:44:26.341+0000 7f05e89cc200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-02-07T12:44:26.538 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:26 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:44:26.536+0000 7f05e89cc200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-02-07T12:44:27.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:26 smithi019 ceph-mon[24574]: from='client.15390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:44:27.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:26 smithi019 ceph-mon[24574]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:44:27.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:26 smithi019 ceph-mon[24574]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-02-07T12:44:27.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:26 smithi019 ceph-mon[24574]: Cluster is now healthy 2024-02-07T12:44:27.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:26 smithi019 ceph-mon[29267]: from='client.15390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:44:27.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:26 smithi019 ceph-mon[29267]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:44:27.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:26 smithi019 ceph-mon[29267]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-02-07T12:44:27.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:26 smithi019 ceph-mon[29267]: Cluster is now healthy 2024-02-07T12:44:27.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:26 smithi172 ceph-mon[25457]: from='client.15390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:44:27.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:26 smithi172 ceph-mon[25457]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:44:27.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:26 smithi172 ceph-mon[25457]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-02-07T12:44:27.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:26 smithi172 ceph-mon[25457]: Cluster is now healthy 2024-02-07T12:44:27.869 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:44:27.541+0000 7f05e89cc200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-02-07T12:44:27.869 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:44:27.675+0000 7f05e89cc200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-02-07T12:44:28.326 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:27 smithi172 ceph-mon[25457]: mgrmap e41: y(active, since 4s) 2024-02-07T12:44:28.326 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:28 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:44:28.025+0000 7f05e89cc200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-02-07T12:44:28.326 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:28 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:44:28.118+0000 7f05e89cc200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-02-07T12:44:28.326 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:28 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:44:28.324+0000 7f05e89cc200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:44:28.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:27 smithi019 ceph-mon[24574]: mgrmap e41: y(active, since 4s) 2024-02-07T12:44:28.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:27 smithi019 ceph-mon[29267]: mgrmap e41: y(active, since 4s) 2024-02-07T12:44:28.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:28 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:44:28.511+0000 7f05e89cc200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-02-07T12:44:29.095 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:28 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:44:28.817+0000 7f05e89cc200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:44:29.346 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:29 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:44:29.093+0000 7f05e89cc200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-02-07T12:44:29.346 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:29 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:44:29.344+0000 7f05e89cc200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-02-07T12:44:29.346 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:29 smithi172 ceph-mon[25457]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:44:29.346 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:29 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:29.347 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:29 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:29.347 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:29 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:29.347 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:29 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:29.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:29 smithi019 ceph-mon[24574]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:44:29.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:29 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:29.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:29 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:29.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:29 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:29.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:29 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:29.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:29 smithi019 ceph-mon[29267]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:44:29.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:29 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:29.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:29 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:29.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:29 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:29.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:29 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:30.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:30 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:44:30.203+0000 7f05e89cc200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-02-07T12:44:31.119 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:30 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:44:30.815+0000 7f05e89cc200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-02-07T12:44:31.119 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:30 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:44:30.922+0000 7f05e89cc200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-02-07T12:44:31.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:30 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:31.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:30 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:31.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:30 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:31.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:30 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:31.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:30 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:44:31.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:30 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:44:31.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:30 smithi172 ceph-mon[25457]: Updating smithi019:/etc/ceph/ceph.conf 2024-02-07T12:44:31.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:30 smithi172 ceph-mon[25457]: Updating smithi172:/etc/ceph/ceph.conf 2024-02-07T12:44:31.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:30 smithi172 ceph-mon[25457]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:44:31.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:30 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:44:31.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:30 smithi172 ceph-mon[25457]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:44:31.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:30 smithi172 ceph-mon[25457]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:44:31.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:30 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:31.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:30 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:31.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:31.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:31.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:31.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:31.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:44:31.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:44:31.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[29267]: Updating smithi019:/etc/ceph/ceph.conf 2024-02-07T12:44:31.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[29267]: Updating smithi172:/etc/ceph/ceph.conf 2024-02-07T12:44:31.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[29267]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:44:31.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:44:31.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[29267]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:44:31.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[29267]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:44:31.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:31.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:31.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:31.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:31.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:31.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:31.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:44:31.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:44:31.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[24574]: Updating smithi019:/etc/ceph/ceph.conf 2024-02-07T12:44:31.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[24574]: Updating smithi172:/etc/ceph/ceph.conf 2024-02-07T12:44:31.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[24574]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:44:31.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:44:31.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[24574]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:44:31.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[24574]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:44:31.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:31.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:30 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:31.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:31 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:44:31.145+0000 7f05e89cc200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-02-07T12:44:31.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:31 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:44:31] ENGINE Bus STARTING 2024-02-07T12:44:31.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:31 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: CherryPy Checker: 2024-02-07T12:44:31.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:31 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: The Application mounted at '' has an empty config. 2024-02-07T12:44:31.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:31 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: 2024-02-07T12:44:31.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:31 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:44:31] ENGINE Serving on http://:::9283 2024-02-07T12:44:31.620 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:31 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[45644]: [07/Feb/2024:12:44:31] ENGINE Bus STARTED 2024-02-07T12:44:32.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:31 smithi172 ceph-mon[25457]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:44:32.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:31 smithi172 ceph-mon[25457]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:44:32.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:31 smithi172 ceph-mon[25457]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:44:32.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:31 smithi172 ceph-mon[25457]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:44:32.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:31 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:32.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:31 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:32.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:31 smithi172 ceph-mon[25457]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 20 op/s 2024-02-07T12:44:32.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:31 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:32.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:31 smithi172 ceph-mon[25457]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-02-07T12:44:32.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:31 smithi172 ceph-mon[25457]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:44:32.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:31 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:44:32.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:31 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:44:32.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:31 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:44:32.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:31 smithi172 ceph-mon[25457]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:44:32.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:31 smithi172 ceph-mon[25457]: Standby manager daemon x started 2024-02-07T12:44:32.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:31 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.172:0/148501334' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-02-07T12:44:32.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:31 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.172:0/148501334' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:44:32.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:31 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.172:0/148501334' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-02-07T12:44:32.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:31 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.172:0/148501334' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:44:32.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[24574]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:44:32.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[24574]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:44:32.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[24574]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:44:32.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[24574]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:44:32.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:32.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:32.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[24574]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 20 op/s 2024-02-07T12:44:32.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:32.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[24574]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-02-07T12:44:32.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[24574]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:44:32.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:44:32.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:44:32.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:44:32.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[24574]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:44:32.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[24574]: Standby manager daemon x started 2024-02-07T12:44:32.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.172:0/148501334' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-02-07T12:44:32.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.172:0/148501334' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:44:32.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.172:0/148501334' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-02-07T12:44:32.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.172:0/148501334' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:44:32.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[29267]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:44:32.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[29267]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:44:32.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[29267]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:44:32.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[29267]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:44:32.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:32.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:32.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[29267]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 20 op/s 2024-02-07T12:44:32.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:32.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[29267]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-02-07T12:44:32.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[29267]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:44:32.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:44:32.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:44:32.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:44:32.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[29267]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:44:32.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[29267]: Standby manager daemon x started 2024-02-07T12:44:32.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/148501334' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-02-07T12:44:32.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/148501334' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:44:32.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/148501334' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-02-07T12:44:32.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:31 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/148501334' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:44:32.908 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:44:32.754+0000 7eff60d5c700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:44:32.908 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:44:32.909 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:44:32.909 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:44:32.909 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:44:32.909 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:32.909 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:32.909 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:44:32.909 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:44:32.909 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:44:32.909 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:44:32.909 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:44:32.909 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:44:32.909 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:44:32.909 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:44:32.909 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:44:32.909 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:44:32.909 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:44:32.910 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:44:32.910 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:32.910 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:32.910 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:44:32.910 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:44:32.910 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:44:32.910 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:44:32.911 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:44:32.911 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:44:32.911 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:44:32.911 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:44:32.911 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:44:32.911 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:44:32.911 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:44:32.911 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:44:32.911 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:44:32.911 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:44:32.911 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:44:32.911 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:44:32.911 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:44:32.911 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:44:32.911 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:44:32.911 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:44:32.912 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:44:32.912 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:44:32.912 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:32.912 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:32.912 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:44:32.912 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:44:32.912 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:44:32.912 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:44:32.912 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:44:32.912 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:44:32.912 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:44:32.912 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:44:32.912 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:44:32.912 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:44:32.913 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:44:32.913 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:44:32.913 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:32.913 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:33.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-mon[29267]: mgrmap e42: y(active, since 8s), standbys: x 2024-02-07T12:44:33.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-02-07T12:44:33.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-mon[24574]: mgrmap e42: y(active, since 8s), standbys: x 2024-02-07T12:44:33.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:32 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-02-07T12:44:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:32 smithi172 ceph-mon[25457]: mgrmap e42: y(active, since 8s), standbys: x 2024-02-07T12:44:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:32 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-02-07T12:44:34.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:34 smithi019 ceph-mon[24574]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:44:34.594 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:44:34.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:44:34.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:44:34.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:44:34.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:34.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:34.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:44:34.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:44:34.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:44:34.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:44:34.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:44:34.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:44:34.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:44:34.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:44:34.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:44:34.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:44:34.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:44:34.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:44:34.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:34.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:34.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:44:34.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:44:34.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:44:34.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:44:34.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:44:34.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:44:34.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:44:34.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:44:34.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:44:34.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:44:34.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:44:34.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:44:34.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:44:34.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:44:34.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:44:34.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:44:34.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:44:34.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:44:34.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:44:34.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:44:34.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:44:34.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:44:34.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:34.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:34.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:44:34.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:44:34.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:44:34.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:44:34.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:44:34.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:44:34.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:44:34.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:44:34.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:44:34.598 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:34 smithi019 ceph-mon[29267]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:44:34.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:44:34.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:44:34.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:44:34.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:44:34.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:34.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:34.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:44:34.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:44:34.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:44:34.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:44:34.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:44:34.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:44:34.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:44:34.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:44:34.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:44:34.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:44:34.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:44:34.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:44:34.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:34.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:34.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:44:34.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:44:34.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:44:34.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:44:34.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:44:34.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:44:34.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:44:34.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:44:34.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:44:34.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:44:34.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:44:34.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:44:34.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:44:34.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:44:34.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:44:34.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:44:34.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:44:34.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:44:34.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:44:34.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:44:34.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:44:34.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:44:34.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:34.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:34.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:44:34.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:44:34.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:44:34.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:44:34.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:44:34.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:44:34.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:44:34.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:44:34.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:44:34.603 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:44:34.603 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:44:34.603 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:44:34.603 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:34.603 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:34.603 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:34 smithi019 ceph-mon[24574]: Reconfiguring prometheus.a (dependencies changed)... 2024-02-07T12:44:34.603 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:34 smithi019 ceph-mon[24574]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 0 B/s wr, 15 op/s 2024-02-07T12:44:34.603 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:34 smithi019 ceph-mon[24574]: Reconfiguring daemon prometheus.a on smithi172 2024-02-07T12:44:34.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:44:34.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:44:34.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:44:34.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:34.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:34.604 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:34 smithi019 ceph-mon[29267]: Reconfiguring prometheus.a (dependencies changed)... 2024-02-07T12:44:34.604 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:34 smithi019 ceph-mon[29267]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 0 B/s wr, 15 op/s 2024-02-07T12:44:34.604 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:34 smithi019 ceph-mon[29267]: Reconfiguring daemon prometheus.a on smithi172 2024-02-07T12:44:34.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:34 smithi172 ceph-mon[25457]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:44:34.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:44:34.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:44:34.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:44:34.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:44:34.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:34.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:34.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:44:34.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:44:34.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:44:34.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:44:34.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:44:34.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:44:34.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:44:34.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:44:34.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:44:34.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:44:34.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:44:34.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:44:34.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:34.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:34.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:44:34.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:44:34.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:44:34.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:44:34.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:44:34.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:44:34.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:44:34.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:44:34.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:44:34.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:44:34.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:44:34.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:44:34.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:44:34.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:44:34.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:44:34.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:44:34.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:44:34.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:44:34.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:44:34.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:44:34.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:44:34.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:44:34.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:34.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:34.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:44:34.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:44:34.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:44:34.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:44:34.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:44:34.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:44:34.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:44:34.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:44:34.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:44:34.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:44:34.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:44:34.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:44:34.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:34.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:34.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:34 smithi172 ceph-mon[25457]: Reconfiguring prometheus.a (dependencies changed)... 2024-02-07T12:44:34.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:34 smithi172 ceph-mon[25457]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 0 B/s wr, 15 op/s 2024-02-07T12:44:34.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:34 smithi172 ceph-mon[25457]: Reconfiguring daemon prometheus.a on smithi172 2024-02-07T12:44:36.552 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:36 smithi172 systemd[1]: Stopping Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:44:36.552 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:36 smithi172 bash[60210]: Error: no container with name or ID "ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus.a" found: no such container 2024-02-07T12:44:36.553 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:36 smithi172 ceph-mon[25457]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-02-07T12:44:36.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:36 smithi019 ceph-mon[29267]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-02-07T12:44:36.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:36 smithi019 ceph-mon[24574]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-02-07T12:44:36.844 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:36 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:44:36.550Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-02-07T12:44:36.844 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:36 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:44:36.550Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-02-07T12:44:36.844 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:36 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:44:36.550Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-02-07T12:44:36.844 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:36 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:44:36.550Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-02-07T12:44:36.844 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:36 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:44:36.550Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-02-07T12:44:36.845 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:36 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:44:36.550Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-02-07T12:44:36.845 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:36 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:44:36.550Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-02-07T12:44:36.845 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:36 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:44:36.550Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-02-07T12:44:36.845 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:36 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:44:36.550Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-02-07T12:44:36.845 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:36 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:44:36.551Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-02-07T12:44:36.845 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:36 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:44:36.551Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-02-07T12:44:36.845 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:36 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[57430]: ts=2024-02-07T12:44:36.551Z caller=main.go:1066 level=info msg="See you next time!" 2024-02-07T12:44:36.845 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:36 smithi172 podman[60223]: 2024-02-07 12:44:36.560557125 +0000 UTC m=+0.114352371 container died 6b7568bd4b22de5ab4b12549d05e2479331daa41fdac2c5c0545065838b6dabd (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:44:37.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:36 smithi172 podman[60223]: 2024-02-07 12:44:36.932950573 +0000 UTC m=+0.486745816 container cleanup 6b7568bd4b22de5ab4b12549d05e2479331daa41fdac2c5c0545065838b6dabd (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:44:37.119 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:36 smithi172 bash[60223]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a 2024-02-07T12:44:37.120 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:36 smithi172 bash[60250]: Error: no container with name or ID "ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus.a" found: no such container 2024-02-07T12:44:37.542 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:37 smithi172 podman[60237]: 2024-02-07 12:44:37.183882394 +0000 UTC m=+0.624895513 container remove 6b7568bd4b22de5ab4b12549d05e2479331daa41fdac2c5c0545065838b6dabd (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:44:37.542 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:37 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@prometheus.a.service: Deactivated successfully. 2024-02-07T12:44:37.542 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:37 smithi172 systemd[1]: Stopped Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:44:37.542 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:37 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@prometheus.a.service: Consumed 1.310s CPU time. 2024-02-07T12:44:37.542 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:37 smithi172 systemd[1]: Starting Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:44:37.542 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:37 smithi172 podman[60318]: 2024-02-07 12:44:37.317729108 +0000 UTC m=+0.019784530 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-02-07T12:44:37.542 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:37 smithi172 podman[60318]: 2024-02-07 12:44:37.418209822 +0000 UTC m=+0.120265244 container create 80e3dbe199442a6bda4f535b03f55ecf18b609ab228f1269f45a0c29c9da3333 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:44:37.869 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:37 smithi172 podman[60318]: 2024-02-07 12:44:37.662972392 +0000 UTC m=+0.365027826 container init 80e3dbe199442a6bda4f535b03f55ecf18b609ab228f1269f45a0c29c9da3333 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:44:37.869 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:37 smithi172 podman[60318]: 2024-02-07 12:44:37.665685481 +0000 UTC m=+0.367740917 container start 80e3dbe199442a6bda4f535b03f55ecf18b609ab228f1269f45a0c29c9da3333 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:44:37.870 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:44:37.692Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-02-07T12:44:37.870 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:44:37.692Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-02-07T12:44:37.870 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:44:37.692Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-02-07T12:44:37.870 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:44:37.692Z caller=main.go:518 level=info host_details="(Linux 5.14.0-418.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Mon Feb 5 23:17:48 UTC 2024 x86_64 smithi172 (none))" 2024-02-07T12:44:37.870 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:44:37.692Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-02-07T12:44:37.870 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:44:37.692Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-02-07T12:44:37.870 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:44:37.694Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-02-07T12:44:37.870 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:44:37.694Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-02-07T12:44:37.870 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:44:37.695Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-02-07T12:44:37.870 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:44:37.698Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-02-07T12:44:37.870 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:44:37.698Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.706µs 2024-02-07T12:44:37.870 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:44:37.698Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-02-07T12:44:37.870 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:37 smithi172 bash[60318]: 80e3dbe199442a6bda4f535b03f55ecf18b609ab228f1269f45a0c29c9da3333 2024-02-07T12:44:37.870 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:37 smithi172 systemd[1]: Started Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:44:38.195 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:37 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:44:37.916Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=5 2024-02-07T12:44:38.195 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:38 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:44:38.140Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=5 2024-02-07T12:44:38.451 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:38 smithi172 ceph-mon[25457]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 0 B/s wr, 47 op/s 2024-02-07T12:44:38.451 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:38 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:38.451 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:38 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:38.451 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:38 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:44:38.451 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:38 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:44:38.451 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:38 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:44:38.194Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=5 2024-02-07T12:44:38.452 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:38 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:44:38.235Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=5 2024-02-07T12:44:38.452 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:38 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:44:38.241Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=5 2024-02-07T12:44:38.452 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:38 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:44:38.242Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=5 2024-02-07T12:44:38.452 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:38 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:44:38.242Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=45.32µs wal_replay_duration=544.380244ms total_replay_duration=544.438585ms 2024-02-07T12:44:38.452 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:38 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:44:38.248Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-02-07T12:44:38.452 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:38 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:44:38.248Z caller=main.go:947 level=info msg="TSDB started" 2024-02-07T12:44:38.452 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:38 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:44:38.248Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-02-07T12:44:38.452 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:38 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:44:38.265Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.236664ms db_storage=766ns remote_storage=1.273µs web_handler=424ns query_engine=748ns scrape=157.711µs scrape_sd=93.889µs notify=13.337µs notify_sd=16.089µs rules=16.583448ms 2024-02-07T12:44:38.452 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:44:38 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:44:38.265Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-02-07T12:44:38.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:38 smithi019 ceph-mon[24574]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 0 B/s wr, 47 op/s 2024-02-07T12:44:38.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:38 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:38.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:38 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:38.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:38 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:44:38.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:38 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:44:38.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:38 smithi019 ceph-mon[29267]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 0 B/s wr, 47 op/s 2024-02-07T12:44:38.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:38 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:38.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:38 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:38.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:38 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:44:38.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:38 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:44:39.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:39 smithi172 ceph-mon[25457]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:44:39.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:39 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:44:39.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:39 smithi019 ceph-mon[24574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:44:39.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:39 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:44:39.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:39 smithi019 ceph-mon[29267]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:44:39.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:39 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:44:40.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:40 smithi019 ceph-mon[29267]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 0 B/s wr, 47 op/s 2024-02-07T12:44:40.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:40 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:44:40.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:40 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:44:40.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:40 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:44:40.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:40 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:44:40.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:40 smithi019 ceph-mon[24574]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 0 B/s wr, 47 op/s 2024-02-07T12:44:40.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:40 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:44:40.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:40 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:44:40.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:40 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:44:40.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:40 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:44:40.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:40 smithi172 ceph-mon[25457]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 0 B/s wr, 47 op/s 2024-02-07T12:44:40.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:40 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:44:40.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:40 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:44:40.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:40 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:44:40.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:40 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:44:41.547 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:41 smithi172 systemd[1]: Stopping Ceph mgr.x for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:44:41.547 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:41 smithi172 ceph-mon[25457]: Upgrade: Updating mgr.x 2024-02-07T12:44:41.547 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:41 smithi172 ceph-mon[25457]: Deploying daemon mgr.x on smithi172 2024-02-07T12:44:41.547 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:41 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:44:41.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:41 smithi019 ceph-mon[24574]: Upgrade: Updating mgr.x 2024-02-07T12:44:41.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:41 smithi019 ceph-mon[24574]: Deploying daemon mgr.x on smithi172 2024-02-07T12:44:41.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:41 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:44:41.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:41 smithi019 ceph-mon[29267]: Upgrade: Updating mgr.x 2024-02-07T12:44:41.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:41 smithi019 ceph-mon[29267]: Deploying daemon mgr.x on smithi172 2024-02-07T12:44:41.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:41 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:44:41.799 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:41 smithi172 podman[60671]: 2024-02-07 12:44:41.544863509 +0000 UTC m=+0.156552806 container died 97644267e529cd5b3f535ef8aef0c569f9aa23b49b12169ac15caabec0172c60 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x, GIT_BRANCH=HEAD, io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, org.label-schema.build-date=20240131) 2024-02-07T12:44:42.058 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:41 smithi172 podman[60671]: 2024-02-07 12:44:41.872129994 +0000 UTC m=+0.483819304 container cleanup 97644267e529cd5b3f535ef8aef0c569f9aa23b49b12169ac15caabec0172c60 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131) 2024-02-07T12:44:42.058 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:41 smithi172 bash[60671]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x 2024-02-07T12:44:42.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:42 smithi172 ceph-mon[25457]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 99 KiB/s rd, 0 B/s wr, 143 op/s 2024-02-07T12:44:42.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:42 smithi172 podman[60685]: 2024-02-07 12:44:42.056741407 +0000 UTC m=+0.514131570 container remove 97644267e529cd5b3f535ef8aef0c569f9aa23b49b12169ac15caabec0172c60 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , io.buildah.version=1.33.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240131) 2024-02-07T12:44:42.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:42 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-02-07T12:44:42.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:42 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mgr.x.service: Failed with result 'exit-code'. 2024-02-07T12:44:42.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:42 smithi172 systemd[1]: Stopped Ceph mgr.x for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:44:42.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:42 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mgr.x.service: Consumed 57.504s CPU time. 2024-02-07T12:44:42.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:42 smithi019 ceph-mon[24574]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 99 KiB/s rd, 0 B/s wr, 143 op/s 2024-02-07T12:44:42.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:42 smithi019 ceph-mon[29267]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 99 KiB/s rd, 0 B/s wr, 143 op/s 2024-02-07T12:44:42.751 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:42 smithi172 systemd[1]: Starting Ceph mgr.x for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:44:42.751 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:42 smithi172 podman[60819]: 2024-02-07 12:44:42.60610856 +0000 UTC m=+0.018514585 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:44:42.751 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:42 smithi172 podman[60819]: 2024-02-07 12:44:42.749669733 +0000 UTC m=+0.162075748 container create 5aa0c04fdfd76ef55235a01bce705de9af99c7dace7efe18bd820d6f2c023853 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-02-07T12:44:43.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:43 smithi172 podman[60819]: 2024-02-07 12:44:43.018718986 +0000 UTC m=+0.431125014 container init 5aa0c04fdfd76ef55235a01bce705de9af99c7dace7efe18bd820d6f2c023853 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:44:43.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:43 smithi172 podman[60819]: 2024-02-07 12:44:43.021311057 +0000 UTC m=+0.433717075 container start 5aa0c04fdfd76ef55235a01bce705de9af99c7dace7efe18bd820d6f2c023853 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-02-07T12:44:43.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:43 smithi172 ceph-mgr[60839]: -- 172.21.15.172:0/3403485118 <== mon.2 v2:172.21.15.172:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f1f9d3d520 con 0x55f1f9d12c00 2024-02-07T12:44:43.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:43 smithi172 bash[60819]: 5aa0c04fdfd76ef55235a01bce705de9af99c7dace7efe18bd820d6f2c023853 2024-02-07T12:44:43.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:43 smithi172 systemd[1]: Started Ceph mgr.x for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:44:43.871 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:43 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:44:43.590+0000 7f2aed6b7200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-02-07T12:44:43.871 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:43 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:44:43.690+0000 7f2aed6b7200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-02-07T12:44:44.188 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:44:43] "GET /metrics HTTP/1.1" 200 37986 "" "Prometheus/2.33.4" 2024-02-07T12:44:44.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:44 smithi172 ceph-mon[25457]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-02-07T12:44:44.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:44 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:44.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:44 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:44.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:44 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:44:44.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:44 smithi019 ceph-mon[24574]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-02-07T12:44:44.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:44 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:44.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:44 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:44.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:44 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:44:44.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:44 smithi019 ceph-mon[29267]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-02-07T12:44:44.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:44 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:44.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:44 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:44.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:44 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:44:44.642 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:44 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:44:44.382+0000 7f2aed6b7200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-02-07T12:44:44.642 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:44 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:44:44.484+0000 7f2aed6b7200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-02-07T12:44:44.952 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:44 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:44:44.638+0000 7f2aed6b7200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-02-07T12:44:44.953 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:44 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:44:44.742+0000 7f2aed6b7200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-02-07T12:44:44.953 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:44 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:44:44.838+0000 7f2aed6b7200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-02-07T12:44:45.241 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:45 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:44:45.113+0000 7f2aed6b7200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-02-07T12:44:45.242 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:45 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:44:45.238+0000 7f2aed6b7200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-02-07T12:44:45.605 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:45 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:44:45.340+0000 7f2aed6b7200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-02-07T12:44:45.605 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:45 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:44:45.452+0000 7f2aed6b7200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-02-07T12:44:45.869 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:45 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:44:45.637+0000 7f2aed6b7200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-02-07T12:44:46.482 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:46 smithi172 ceph-mon[25457]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-02-07T12:44:46.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:46 smithi019 ceph-mon[29267]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-02-07T12:44:46.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:46 smithi019 ceph-mon[24574]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-02-07T12:44:46.801 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:46 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:44:46.662+0000 7f2aed6b7200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-02-07T12:44:46.801 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:46 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:44:46.798+0000 7f2aed6b7200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-02-07T12:44:47.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:44:47.156+0000 7f2aed6b7200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-02-07T12:44:47.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:44:47.250+0000 7f2aed6b7200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-02-07T12:44:47.869 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:44:47.452+0000 7f2aed6b7200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:44:47.869 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:44:47.656+0000 7f2aed6b7200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-02-07T12:44:48.249 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:47 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:44:47.968+0000 7f2aed6b7200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:44:48.250 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:48 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:44:48.246+0000 7f2aed6b7200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-02-07T12:44:48.250 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:48 smithi172 ceph-mon[25457]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-02-07T12:44:48.250 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:48 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:48.250 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:48 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:48.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:48 smithi019 ceph-mon[24574]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-02-07T12:44:48.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:48 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:48.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:48 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:48.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:48 smithi019 ceph-mon[29267]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-02-07T12:44:48.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:48 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:48.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:48 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:48.869 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:48 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:44:48.503+0000 7f2aed6b7200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-02-07T12:44:49.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:49 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:44:49.377+0000 7f2aed6b7200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-02-07T12:44:49.976 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:49 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:49.976 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:49 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:49.976 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:49 smithi172 ceph-mon[25457]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 0 B/s wr, 87 op/s 2024-02-07T12:44:50.036 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:49 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:50.036 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:49 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:50.036 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:49 smithi019 ceph-mon[24574]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 0 B/s wr, 87 op/s 2024-02-07T12:44:50.037 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:49 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:50.037 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:49 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:50.037 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:49 smithi019 ceph-mon[29267]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 0 B/s wr, 87 op/s 2024-02-07T12:44:50.304 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:49 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:44:49.972+0000 7f2aed6b7200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-02-07T12:44:50.305 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:50 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:44:50.077+0000 7f2aed6b7200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-02-07T12:44:50.565 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:50 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:44:50.300+0000 7f2aed6b7200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-02-07T12:44:50.565 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:50 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: [07/Feb/2024:12:44:50] ENGINE Bus STARTING 2024-02-07T12:44:50.565 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:50 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: CherryPy Checker: 2024-02-07T12:44:50.565 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:50 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: The Application mounted at '' has an empty config. 2024-02-07T12:44:50.566 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:50 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:44:50.566 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:50 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: [07/Feb/2024:12:44:50] ENGINE Serving on http://:::9283 2024-02-07T12:44:50.566 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:44:50 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: [07/Feb/2024:12:44:50] ENGINE Bus STARTED 2024-02-07T12:44:50.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:50 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:44:50.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:50 smithi172 ceph-mon[25457]: Standby manager daemon x restarted 2024-02-07T12:44:50.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:50 smithi172 ceph-mon[25457]: Standby manager daemon x started 2024-02-07T12:44:50.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:50 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.172:0/786011186' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-02-07T12:44:50.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:50 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.172:0/786011186' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:44:50.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:50 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.172:0/786011186' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-02-07T12:44:50.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:50 smithi172 ceph-mon[25457]: from='mgr.? 172.21.15.172:0/786011186' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:44:51.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:50 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:44:51.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:50 smithi019 ceph-mon[24574]: Standby manager daemon x restarted 2024-02-07T12:44:51.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:50 smithi019 ceph-mon[24574]: Standby manager daemon x started 2024-02-07T12:44:51.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:50 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.172:0/786011186' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-02-07T12:44:51.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:50 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.172:0/786011186' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:44:51.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:50 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.172:0/786011186' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-02-07T12:44:51.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:50 smithi019 ceph-mon[24574]: from='mgr.? 172.21.15.172:0/786011186' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:44:51.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:50 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:44:51.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:50 smithi019 ceph-mon[29267]: Standby manager daemon x restarted 2024-02-07T12:44:51.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:50 smithi019 ceph-mon[29267]: Standby manager daemon x started 2024-02-07T12:44:51.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:50 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/786011186' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-02-07T12:44:51.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:50 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/786011186' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:44:51.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:50 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/786011186' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-02-07T12:44:51.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:50 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/786011186' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:44:52.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:51 smithi019 ceph-mon[24574]: mgrmap e43: y(active, since 27s), standbys: x 2024-02-07T12:44:52.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:51 smithi019 ceph-mon[24574]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 0 B/s wr, 87 op/s 2024-02-07T12:44:52.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:51 smithi019 ceph-mon[29267]: mgrmap e43: y(active, since 27s), standbys: x 2024-02-07T12:44:52.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:51 smithi019 ceph-mon[29267]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 0 B/s wr, 87 op/s 2024-02-07T12:44:52.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:51 smithi172 ceph-mon[25457]: mgrmap e43: y(active, since 27s), standbys: x 2024-02-07T12:44:52.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:51 smithi172 ceph-mon[25457]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 0 B/s wr, 87 op/s 2024-02-07T12:44:53.181 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:53 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:53.182 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:53 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:53.182 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:53 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:44:53.182 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:53 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:44:53.182 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:53 smithi019 ceph-mon[29267]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 906 B/s rd, 0 op/s 2024-02-07T12:44:53.183 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:53 smithi019 ceph-mon[29267]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:44:53.183 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:53 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:53.183 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:53 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:44:53.183 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:53 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:44:53.183 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:53 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:44:53.183 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:53 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:53.183 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:53 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:53.183 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:53 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:44:53.183 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:53 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:44:53.183 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:53 smithi019 ceph-mon[24574]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 906 B/s rd, 0 op/s 2024-02-07T12:44:53.183 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:53 smithi019 ceph-mon[24574]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:44:53.184 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:53 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:53.184 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:53 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:44:53.184 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:53 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:44:53.184 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:53 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:44:53.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:53 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:53.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:53 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:53.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:53 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:44:53.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:53 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:44:53.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:53 smithi172 ceph-mon[25457]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 906 B/s rd, 0 op/s 2024-02-07T12:44:53.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:53 smithi172 ceph-mon[25457]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:44:53.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:53 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:53.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:53 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:44:53.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:53 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:44:53.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:53 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:44:54.212 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:44:53] "GET /metrics HTTP/1.1" 200 37986 "" "Prometheus/2.33.4" 2024-02-07T12:44:54.212 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:44:54.127+0000 7eff60d5c700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:44:54.212 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:44:54.212 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:44:54.212 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:44:54.212 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:44:54.212 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:54.212 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:54.212 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:44:54.213 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:44:54.213 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:44:54.213 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:44:54.213 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:44:54.213 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:44:54.213 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:44:54.213 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:44:54.213 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:44:54.213 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:44:54.213 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:44:54.214 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:44:54.214 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:54.214 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:54.214 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:44:54.214 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:44:54.214 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:44:54.214 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:44:54.214 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:44:54.214 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:44:54.214 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:44:54.214 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:44:54.215 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:44:54.215 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:44:54.215 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:44:54.215 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:44:54.215 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:44:54.215 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:44:54.215 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:44:54.215 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:44:54.215 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:44:54.215 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:44:54.215 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:44:54.215 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:44:54.215 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:44:54.215 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:44:54.215 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:54.216 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:54.216 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:44:54.216 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:44:54.216 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:44:54.216 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:44:54.216 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:44:54.216 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:44:54.216 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:44:54.216 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:44:54.216 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:44:54.216 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:44:54.216 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:44:54.216 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:44:54.217 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:54.217 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:54.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[29267]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:44:54.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[29267]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:44:54.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:44:54.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:44:54.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:54.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:54.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-02-07T12:44:54.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-02-07T12:44:54.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-02-07T12:44:54.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-02-07T12:44:54.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-02-07T12:44:54.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-02-07T12:44:54.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:54.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:54.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:54.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:54.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:54.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:54.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:54.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:54.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:54.599 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:54.599 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:54.599 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:54.600 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[24574]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:44:54.600 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[24574]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:44:54.600 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:44:54.600 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:44:54.600 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:54.600 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:54.600 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-02-07T12:44:54.600 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-02-07T12:44:54.600 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-02-07T12:44:54.600 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-02-07T12:44:54.600 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-02-07T12:44:54.600 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-02-07T12:44:54.601 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:54.601 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:54.601 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:54.601 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:54.601 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:54.601 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:54.601 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:54.601 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:54.601 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:54.601 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:54.601 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:54.601 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:54 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:54.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:54 smithi172 ceph-mon[25457]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:44:54.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:54 smithi172 ceph-mon[25457]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:44:54.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:54 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:44:54.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:54 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:44:54.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:54 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:54.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:54 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:54.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:54 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-02-07T12:44:54.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:54 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-02-07T12:44:54.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:54 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-02-07T12:44:54.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:54 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-02-07T12:44:54.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:54 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-02-07T12:44:54.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:54 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-02-07T12:44:54.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:54 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:54.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:54 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:54.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:54 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:54.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:54 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:54.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:54 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:54.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:54 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:54.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:54 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:54.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:54 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:54.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:54 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:54.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:54 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:54.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:54 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:54.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:54 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:55.416 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:55 smithi019 ceph-mon[24574]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:44:55.416 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:44:55.417 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:44:55.417 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:44:55.417 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:44:55.417 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:55.417 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:55.417 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:44:55.417 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:44:55.417 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:44:55.417 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:44:55.417 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:44:55.417 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:44:55.417 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:44:55.417 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:44:55.417 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:44:55.418 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:44:55.418 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:44:55.418 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:44:55.418 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:55.418 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:55.418 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:44:55.418 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:44:55.418 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:44:55.418 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:44:55.418 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:44:55.418 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:44:55.418 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:44:55.418 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:44:55.418 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:44:55.418 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:44:55.419 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:44:55.419 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:44:55.419 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:44:55.419 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:44:55.419 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:44:55.419 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:44:55.419 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:44:55.419 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:44:55.419 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:44:55.419 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:44:55.419 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:44:55.419 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:44:55.419 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:55.419 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:55.420 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:44:55.420 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:44:55.420 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:44:55.420 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:44:55.420 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:44:55.420 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:44:55.420 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:44:55.420 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:44:55.420 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:44:55.421 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:55 smithi019 ceph-mon[29267]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:44:55.421 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:44:55.421 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:44:55.421 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:44:55.421 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:44:55.421 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:55.421 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:55.421 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:44:55.421 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:44:55.421 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:44:55.421 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:44:55.421 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:44:55.421 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:44:55.421 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:44:55.422 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:44:55.422 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:44:55.422 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:44:55.422 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:44:55.422 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:44:55.422 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:55.422 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:55.422 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:44:55.422 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:44:55.422 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:44:55.422 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:44:55.422 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:44:55.422 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:44:55.422 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:44:55.423 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:44:55.423 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:44:55.423 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:44:55.423 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:44:55.423 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:44:55.423 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:44:55.423 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:44:55.423 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:44:55.423 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:44:55.423 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:44:55.423 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:44:55.423 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:44:55.423 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:44:55.423 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:44:55.423 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:44:55.424 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:55.424 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:55.424 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:44:55.424 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:44:55.424 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:44:55.424 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:44:55.424 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:44:55.424 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:44:55.424 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:44:55.424 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:44:55.424 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:44:55.424 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:44:55.425 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:44:55.425 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:44:55.425 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:55.425 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:55.425 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:55 smithi019 ceph-mon[24574]: Upgrade: Setting container_image for all mgr 2024-02-07T12:44:55.425 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:55 smithi019 ceph-mon[24574]: Upgrade: Setting container_image for all crash 2024-02-07T12:44:55.425 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:55 smithi019 ceph-mon[24574]: Upgrade: Setting container_image for all mds 2024-02-07T12:44:55.425 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:55 smithi019 ceph-mon[24574]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:44:55.425 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:55 smithi019 ceph-mon[24574]: Upgrade: Setting container_image for all rbd-mirror 2024-02-07T12:44:55.425 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:55 smithi019 ceph-mon[24574]: Upgrade: Setting container_image for all cephfs-mirror 2024-02-07T12:44:55.425 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:55 smithi019 ceph-mon[24574]: Upgrade: Setting container_image for all ceph-exporter 2024-02-07T12:44:55.425 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:55 smithi019 ceph-mon[24574]: Upgrade: Setting container_image for all nfs 2024-02-07T12:44:55.425 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:55 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:55.426 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:55 smithi019 ceph-mon[24574]: Upgrade: Setting container_image for all nvmeof 2024-02-07T12:44:55.426 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:55 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:55.426 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:44:55.426 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:44:55.426 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:44:55.426 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:55.426 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:55.426 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:55 smithi019 ceph-mon[29267]: Upgrade: Setting container_image for all mgr 2024-02-07T12:44:55.426 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:55 smithi019 ceph-mon[29267]: Upgrade: Setting container_image for all crash 2024-02-07T12:44:55.426 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:55 smithi019 ceph-mon[29267]: Upgrade: Setting container_image for all mds 2024-02-07T12:44:55.426 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:55 smithi019 ceph-mon[29267]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:44:55.426 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:55 smithi019 ceph-mon[29267]: Upgrade: Setting container_image for all rbd-mirror 2024-02-07T12:44:55.427 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:55 smithi019 ceph-mon[29267]: Upgrade: Setting container_image for all cephfs-mirror 2024-02-07T12:44:55.427 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:55 smithi019 ceph-mon[29267]: Upgrade: Setting container_image for all ceph-exporter 2024-02-07T12:44:55.427 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:55 smithi019 ceph-mon[29267]: Upgrade: Setting container_image for all nfs 2024-02-07T12:44:55.427 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:55 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:55.427 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:55 smithi019 ceph-mon[29267]: Upgrade: Setting container_image for all nvmeof 2024-02-07T12:44:55.427 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:55 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:55.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:55 smithi172 ceph-mon[25457]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:44:55.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:44:55.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:44:55.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:44:55.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:44:55.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:55.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:55.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:44:55.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:44:55.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:44:55.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:44:55.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:44:55.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:44:55.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:44:55.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:44:55.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:44:55.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:44:55.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:44:55.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:44:55.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:55.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:55.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:44:55.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:44:55.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:44:55.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:44:55.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:44:55.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:44:55.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:44:55.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:44:55.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:44:55.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:44:55.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:44:55.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:44:55.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:44:55.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:44:55.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:44:55.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:44:55.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:44:55.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:44:55.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:44:55.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:44:55.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:44:55.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:44:55.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:55.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:55.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:44:55.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:44:55.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:44:55.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:44:55.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:44:55.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:44:55.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:44:55.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:44:55.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:44:55.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:44:55.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:44:55.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:44:55.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:44:55.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:44:55.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:55 smithi172 ceph-mon[25457]: Upgrade: Setting container_image for all mgr 2024-02-07T12:44:55.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:55 smithi172 ceph-mon[25457]: Upgrade: Setting container_image for all crash 2024-02-07T12:44:55.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:55 smithi172 ceph-mon[25457]: Upgrade: Setting container_image for all mds 2024-02-07T12:44:55.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:55 smithi172 ceph-mon[25457]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:44:55.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:55 smithi172 ceph-mon[25457]: Upgrade: Setting container_image for all rbd-mirror 2024-02-07T12:44:55.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:55 smithi172 ceph-mon[25457]: Upgrade: Setting container_image for all cephfs-mirror 2024-02-07T12:44:55.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:55 smithi172 ceph-mon[25457]: Upgrade: Setting container_image for all ceph-exporter 2024-02-07T12:44:55.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:55 smithi172 ceph-mon[25457]: Upgrade: Setting container_image for all nfs 2024-02-07T12:44:55.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:55 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:55.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:55 smithi172 ceph-mon[25457]: Upgrade: Setting container_image for all nvmeof 2024-02-07T12:44:55.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:55 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:44:56.108 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:44:56.774 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:44:56 smithi019 systemd[1]: Stopping Ceph node-exporter.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:44:56.816 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:44:56.816 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (13m) 28s ago 21m 23.7M - 0.23.0 ba2b418f427c dfaf78b29ab5 2024-02-07T12:44:56.816 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (12m) 9s ago 20m 52.0M - 8.3.5 dad864ee21e9 e753e96fa4f0 2024-02-07T12:44:56.816 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 error 28s ago 20m 57.0M - 3.5 e1d6a67b021e 5113aa77e3f2 2024-02-07T12:44:56.816 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283,8765 running (14s) 9s ago 25m 66.3M - 19.0.0-1252-g44458cce 79957b372307 5aa0c04fdfd7 2024-02-07T12:44:56.816 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (10m) 28s ago 27m 475M - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:44:56.816 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (27m) 28s ago 27m 77.6M 2048M 17.2.0 e1d6a67b021e f3a8a050cf1d 2024-02-07T12:44:56.817 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (25m) 9s ago 25m 57.4M 2048M 17.2.0 e1d6a67b021e 7d62c7c4d075 2024-02-07T12:44:56.817 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (25m) 28s ago 25m 58.5M 2048M 17.2.0 e1d6a67b021e b9825d0db32d 2024-02-07T12:44:56.817 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (13m) 28s ago 21m 18.5M - 1.3.1 1dbe0e931976 9868df14bcc9 2024-02-07T12:44:56.817 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (12m) 9s ago 21m 17.3M - 1.3.1 1dbe0e931976 e066f1871320 2024-02-07T12:44:56.817 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (25m) 28s ago 25m 53.5M 2993M 17.2.0 e1d6a67b021e 87f2560a9b15 2024-02-07T12:44:56.817 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (24m) 28s ago 24m 55.8M 2993M 17.2.0 e1d6a67b021e 0f9a190f0e0d 2024-02-07T12:44:56.817 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (24m) 28s ago 24m 52.4M 2993M 17.2.0 e1d6a67b021e 7db94f25a1aa 2024-02-07T12:44:56.817 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (24m) 28s ago 24m 53.1M 2993M 17.2.0 e1d6a67b021e 0b6b674188ed 2024-02-07T12:44:56.817 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (23m) 9s ago 23m 55.1M 3248M 17.2.0 e1d6a67b021e df72cfd69b3b 2024-02-07T12:44:56.817 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (23m) 9s ago 23m 52.5M 3248M 17.2.0 e1d6a67b021e 3bdb22c07537 2024-02-07T12:44:56.817 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (22m) 9s ago 22m 51.9M 3248M 17.2.0 e1d6a67b021e 7859b651e0d9 2024-02-07T12:44:56.817 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (22m) 9s ago 22m 55.2M 3248M 17.2.0 e1d6a67b021e b301e7c75d73 2024-02-07T12:44:56.817 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (19s) 9s ago 21m 48.9M - 2.33.4 514e6a882f6e 80e3dbe19944 2024-02-07T12:44:56.817 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (20m) 28s ago 20m 77.4M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:44:56.818 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (20m) 9s ago 20m 77.1M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:44:57.031 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:44:56 smithi019 podman[74282]: 2024-02-07 12:44:56.7722883 +0000 UTC m=+0.110897373 container died 9868df14bcc98e518765be8ca65eb64df8aa96b470d7567de6ec2e56ee7421e9 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-02-07T12:44:57.256 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:44:57.256 INFO:teuthology.orchestra.run.smithi019.stdout: "mon": { 2024-02-07T12:44:57.256 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-02-07T12:44:57.256 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:44:57.256 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr": { 2024-02-07T12:44:57.256 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:44:57.256 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:44:57.256 INFO:teuthology.orchestra.run.smithi019.stdout: "osd": { 2024-02-07T12:44:57.256 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-02-07T12:44:57.256 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:44:57.257 INFO:teuthology.orchestra.run.smithi019.stdout: "mds": {}, 2024-02-07T12:44:57.257 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw": { 2024-02-07T12:44:57.257 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:44:57.257 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:44:57.257 INFO:teuthology.orchestra.run.smithi019.stdout: "overall": { 2024-02-07T12:44:57.257 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-02-07T12:44:57.257 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:44:57.257 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:44:57.257 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:44:57.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:57 smithi019 ceph-mon[29267]: Upgrade: Updating node-exporter.a (1/2) 2024-02-07T12:44:57.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:57 smithi019 ceph-mon[29267]: Deploying daemon node-exporter.a on smithi019 2024-02-07T12:44:57.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:57 smithi019 ceph-mon[29267]: from='client.25291 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:44:57.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:57 smithi019 ceph-mon[29267]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:44:57.344 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:44:57 smithi019 podman[74282]: 2024-02-07 12:44:57.136928697 +0000 UTC m=+0.475537769 container cleanup 9868df14bcc98e518765be8ca65eb64df8aa96b470d7567de6ec2e56ee7421e9 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-02-07T12:44:57.345 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:44:57 smithi019 bash[74282]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a 2024-02-07T12:44:57.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:57 smithi019 ceph-mon[24574]: Upgrade: Updating node-exporter.a (1/2) 2024-02-07T12:44:57.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:57 smithi019 ceph-mon[24574]: Deploying daemon node-exporter.a on smithi019 2024-02-07T12:44:57.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:57 smithi019 ceph-mon[24574]: from='client.25291 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:44:57.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:57 smithi019 ceph-mon[24574]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:44:57.604 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:44:57.604 INFO:teuthology.orchestra.run.smithi019.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:44:57.604 INFO:teuthology.orchestra.run.smithi019.stdout: "in_progress": true, 2024-02-07T12:44:57.604 INFO:teuthology.orchestra.run.smithi019.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-02-07T12:44:57.604 INFO:teuthology.orchestra.run.smithi019.stdout: "services_complete": [ 2024-02-07T12:44:57.604 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr" 2024-02-07T12:44:57.604 INFO:teuthology.orchestra.run.smithi019.stdout: ], 2024-02-07T12:44:57.604 INFO:teuthology.orchestra.run.smithi019.stdout: "progress": "2/2 daemons upgraded", 2024-02-07T12:44:57.604 INFO:teuthology.orchestra.run.smithi019.stdout: "message": "Currently upgrading node-exporter daemons", 2024-02-07T12:44:57.604 INFO:teuthology.orchestra.run.smithi019.stdout: "is_paused": false 2024-02-07T12:44:57.604 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:44:57.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:57 smithi172 ceph-mon[25457]: Upgrade: Updating node-exporter.a (1/2) 2024-02-07T12:44:57.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:57 smithi172 ceph-mon[25457]: Deploying daemon node-exporter.a on smithi019 2024-02-07T12:44:57.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:57 smithi172 ceph-mon[25457]: from='client.25291 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:44:57.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:57 smithi172 ceph-mon[25457]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:44:57.782 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:44:57 smithi019 podman[74296]: 2024-02-07 12:44:57.446817672 +0000 UTC m=+0.678573011 container remove 9868df14bcc98e518765be8ca65eb64df8aa96b470d7567de6ec2e56ee7421e9 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-02-07T12:44:57.782 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:44:57 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-02-07T12:44:57.782 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:44:57 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@node-exporter.a.service: Failed with result 'exit-code'. 2024-02-07T12:44:57.782 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:44:57 smithi019 systemd[1]: Stopped Ceph node-exporter.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:44:57.782 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:44:57 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@node-exporter.a.service: Consumed 5.079s CPU time. 2024-02-07T12:44:58.094 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:44:57 smithi019 systemd[1]: Starting Ceph node-exporter.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:44:58.094 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:44:57 smithi019 bash[74475]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-02-07T12:44:58.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:58 smithi019 ceph-mon[29267]: from='client.15411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:44:58.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:58 smithi019 ceph-mon[29267]: from='client.25303 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:44:58.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:58 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2104665413' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:58.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:58 smithi019 ceph-mon[24574]: from='client.15411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:44:58.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:58 smithi019 ceph-mon[24574]: from='client.25303 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:44:58.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:58 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2104665413' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:58.595 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:44:58 smithi019 bash[74475]: Getting image source signatures 2024-02-07T12:44:58.595 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:44:58 smithi019 bash[74475]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-02-07T12:44:58.595 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:44:58 smithi019 bash[74475]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-02-07T12:44:58.595 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:44:58 smithi019 bash[74475]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-02-07T12:44:58.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:58 smithi172 ceph-mon[25457]: from='client.15411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:44:58.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:58 smithi172 ceph-mon[25457]: from='client.25303 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:44:58.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:58 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2104665413' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:44:59.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:59 smithi019 ceph-mon[29267]: from='client.15429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:44:59.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:44:59 smithi019 ceph-mon[29267]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:44:59.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:59 smithi019 ceph-mon[24574]: from='client.15429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:44:59.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:44:59 smithi019 ceph-mon[24574]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:44:59.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:59 smithi172 ceph-mon[25457]: from='client.15429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:44:59.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:44:59 smithi172 ceph-mon[25457]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:45:00.045 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:44:59 smithi019 bash[74475]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-02-07T12:45:00.045 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:44:59 smithi019 bash[74475]: Writing manifest to image destination 2024-02-07T12:45:00.344 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 podman[74475]: 2024-02-07 12:45:00.18515753 +0000 UTC m=+2.308839008 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-02-07T12:45:00.344 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 podman[74475]: 2024-02-07 12:45:00.275637486 +0000 UTC m=+2.399319005 container create c4bdcb1c4d49b53ed683f2d70ce8ef68c15a0fee19ecac17295c5c34bb77533d (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-02-07T12:45:00.845 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 podman[74475]: 2024-02-07 12:45:00.477864387 +0000 UTC m=+2.601545865 container init c4bdcb1c4d49b53ed683f2d70ce8ef68c15a0fee19ecac17295c5c34bb77533d (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-02-07T12:45:00.845 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 podman[74475]: 2024-02-07 12:45:00.480685301 +0000 UTC m=+2.604366779 container start c4bdcb1c4d49b53ed683f2d70ce8ef68c15a0fee19ecac17295c5c34bb77533d (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-02-07T12:45:00.845 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.484Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-02-07T12:45:00.845 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.484Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-02-07T12:45:00.845 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.485Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-02-07T12:45:00.845 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.485Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-02-07T12:45:00.845 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.485Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-02-07T12:45:00.845 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-02-07T12:45:00.845 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-02-07T12:45:00.845 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=arp 2024-02-07T12:45:00.846 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=bcache 2024-02-07T12:45:00.846 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=bonding 2024-02-07T12:45:00.846 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=btrfs 2024-02-07T12:45:00.846 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=conntrack 2024-02-07T12:45:00.846 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=cpu 2024-02-07T12:45:00.846 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-02-07T12:45:00.846 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=diskstats 2024-02-07T12:45:00.846 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=dmi 2024-02-07T12:45:00.846 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=edac 2024-02-07T12:45:00.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=entropy 2024-02-07T12:45:00.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-02-07T12:45:00.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=filefd 2024-02-07T12:45:00.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=filesystem 2024-02-07T12:45:00.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=hwmon 2024-02-07T12:45:00.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=infiniband 2024-02-07T12:45:00.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=ipvs 2024-02-07T12:45:00.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=loadavg 2024-02-07T12:45:00.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=mdadm 2024-02-07T12:45:00.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=meminfo 2024-02-07T12:45:00.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=netclass 2024-02-07T12:45:00.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=netdev 2024-02-07T12:45:00.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=netstat 2024-02-07T12:45:00.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=nfs 2024-02-07T12:45:00.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=nfsd 2024-02-07T12:45:00.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=nvme 2024-02-07T12:45:00.847 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=os 2024-02-07T12:45:00.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-02-07T12:45:00.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=pressure 2024-02-07T12:45:00.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=rapl 2024-02-07T12:45:00.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=schedstat 2024-02-07T12:45:00.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=selinux 2024-02-07T12:45:00.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=sockstat 2024-02-07T12:45:00.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=softnet 2024-02-07T12:45:00.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=stat 2024-02-07T12:45:00.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=tapestats 2024-02-07T12:45:00.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=textfile 2024-02-07T12:45:00.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-02-07T12:45:00.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=time 2024-02-07T12:45:00.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-02-07T12:45:00.848 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=uname 2024-02-07T12:45:00.849 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=vmstat 2024-02-07T12:45:00.849 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=xfs 2024-02-07T12:45:00.849 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=node_exporter.go:117 level=info collector=zfs 2024-02-07T12:45:00.849 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-02-07T12:45:00.849 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a[74526]: ts=2024-02-07T12:45:00.486Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-02-07T12:45:00.849 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 bash[74475]: c4bdcb1c4d49b53ed683f2d70ce8ef68c15a0fee19ecac17295c5c34bb77533d 2024-02-07T12:45:00.849 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:45:00 smithi019 systemd[1]: Started Ceph node-exporter.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:45:01.516 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:01 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:45:01.516 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:01 smithi172 ceph-mon[25457]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:45:01.516 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:01 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:01.516 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:01 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:01.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:01 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:45:01.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:01 smithi019 ceph-mon[24574]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:45:01.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:01 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:01.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:01 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:01.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:01 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:45:01.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:01 smithi019 ceph-mon[29267]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:45:01.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:01 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:01.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:01 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:03.119 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:03 smithi172 systemd[1]: Stopping Ceph node-exporter.b for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:45:03.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:03 smithi019 ceph-mon[29267]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:45:03.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:03 smithi019 ceph-mon[29267]: Upgrade: Updating node-exporter.b (2/2) 2024-02-07T12:45:03.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:03 smithi019 ceph-mon[29267]: Deploying daemon node-exporter.b on smithi172 2024-02-07T12:45:03.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:03 smithi019 ceph-mon[24574]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:45:03.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:03 smithi019 ceph-mon[24574]: Upgrade: Updating node-exporter.b (2/2) 2024-02-07T12:45:03.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:03 smithi019 ceph-mon[24574]: Deploying daemon node-exporter.b on smithi172 2024-02-07T12:45:03.558 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:03 smithi172 podman[62212]: 2024-02-07 12:45:03.316491508 +0000 UTC m=+0.150799411 container died e066f18713209b161990d0825e3d977220e256dc4953a2a54f3ae4fb2f1560c2 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-02-07T12:45:03.558 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:03 smithi172 ceph-mon[25457]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:45:03.558 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:03 smithi172 ceph-mon[25457]: Upgrade: Updating node-exporter.b (2/2) 2024-02-07T12:45:03.558 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:03 smithi172 ceph-mon[25457]: Deploying daemon node-exporter.b on smithi172 2024-02-07T12:45:03.866 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:03 smithi172 podman[62212]: 2024-02-07 12:45:03.655414398 +0000 UTC m=+0.489722308 container cleanup e066f18713209b161990d0825e3d977220e256dc4953a2a54f3ae4fb2f1560c2 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-02-07T12:45:03.866 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:03 smithi172 bash[62212]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b 2024-02-07T12:45:04.119 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:03 smithi172 podman[62226]: 2024-02-07 12:45:03.865213024 +0000 UTC m=+0.552546993 container remove e066f18713209b161990d0825e3d977220e256dc4953a2a54f3ae4fb2f1560c2 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-02-07T12:45:04.119 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:03 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-02-07T12:45:04.119 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:04 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@node-exporter.b.service: Failed with result 'exit-code'. 2024-02-07T12:45:04.119 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:04 smithi172 systemd[1]: Stopped Ceph node-exporter.b for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:45:04.119 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:04 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@node-exporter.b.service: Consumed 5.126s CPU time. 2024-02-07T12:45:04.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:45:03] "GET /metrics HTTP/1.1" 200 37986 "" "Prometheus/2.33.4" 2024-02-07T12:45:04.620 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:04 smithi172 systemd[1]: Starting Ceph node-exporter.b for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:45:04.620 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:04 smithi172 bash[62358]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-02-07T12:45:05.236 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:04 smithi172 bash[62358]: Getting image source signatures 2024-02-07T12:45:05.236 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:04 smithi172 bash[62358]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-02-07T12:45:05.236 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:04 smithi172 bash[62358]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-02-07T12:45:05.236 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:04 smithi172 bash[62358]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-02-07T12:45:05.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:05 smithi019 ceph-mon[24574]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:45:05.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:05 smithi019 ceph-mon[29267]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:45:05.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:05 smithi172 ceph-mon[25457]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:45:06.872 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:06 smithi172 bash[62358]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-02-07T12:45:06.873 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:06 smithi172 bash[62358]: Writing manifest to image destination 2024-02-07T12:45:07.213 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:06 smithi172 podman[62358]: 2024-02-07 12:45:06.893973912 +0000 UTC m=+2.525895503 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-02-07T12:45:07.213 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:06 smithi172 podman[62358]: 2024-02-07 12:45:06.992562071 +0000 UTC m=+2.624483657 container create 04ea81f79d853bc93a320860fb65f18461b68ebef6cc94f5d7a6fb2cf02f41da (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-02-07T12:45:07.554 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-mon[25457]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:45:07.554 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 podman[62358]: 2024-02-07 12:45:07.211731605 +0000 UTC m=+2.843653192 container init 04ea81f79d853bc93a320860fb65f18461b68ebef6cc94f5d7a6fb2cf02f41da (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-02-07T12:45:07.555 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 podman[62358]: 2024-02-07 12:45:07.214664047 +0000 UTC m=+2.846585644 container start 04ea81f79d853bc93a320860fb65f18461b68ebef6cc94f5d7a6fb2cf02f41da (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-02-07T12:45:07.555 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.218Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-02-07T12:45:07.555 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.218Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-02-07T12:45:07.555 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.219Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-02-07T12:45:07.555 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.219Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-02-07T12:45:07.555 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.219Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-02-07T12:45:07.555 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.219Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-02-07T12:45:07.555 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-02-07T12:45:07.556 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=arp 2024-02-07T12:45:07.556 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=bcache 2024-02-07T12:45:07.556 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=bonding 2024-02-07T12:45:07.556 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=btrfs 2024-02-07T12:45:07.556 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=conntrack 2024-02-07T12:45:07.556 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=cpu 2024-02-07T12:45:07.556 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-02-07T12:45:07.557 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=diskstats 2024-02-07T12:45:07.558 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=dmi 2024-02-07T12:45:07.558 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=edac 2024-02-07T12:45:07.558 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=entropy 2024-02-07T12:45:07.558 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-02-07T12:45:07.558 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=filefd 2024-02-07T12:45:07.558 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=filesystem 2024-02-07T12:45:07.559 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=hwmon 2024-02-07T12:45:07.559 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=infiniband 2024-02-07T12:45:07.559 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=ipvs 2024-02-07T12:45:07.559 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=loadavg 2024-02-07T12:45:07.559 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=mdadm 2024-02-07T12:45:07.559 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=meminfo 2024-02-07T12:45:07.559 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=netclass 2024-02-07T12:45:07.559 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=netdev 2024-02-07T12:45:07.559 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=netstat 2024-02-07T12:45:07.559 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=nfs 2024-02-07T12:45:07.560 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=nfsd 2024-02-07T12:45:07.560 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=nvme 2024-02-07T12:45:07.560 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=os 2024-02-07T12:45:07.560 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-02-07T12:45:07.560 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=pressure 2024-02-07T12:45:07.560 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=rapl 2024-02-07T12:45:07.560 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=schedstat 2024-02-07T12:45:07.560 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=selinux 2024-02-07T12:45:07.560 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=sockstat 2024-02-07T12:45:07.561 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=softnet 2024-02-07T12:45:07.561 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=stat 2024-02-07T12:45:07.561 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=tapestats 2024-02-07T12:45:07.561 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=textfile 2024-02-07T12:45:07.561 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-02-07T12:45:07.561 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=time 2024-02-07T12:45:07.561 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-02-07T12:45:07.562 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=uname 2024-02-07T12:45:07.562 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=vmstat 2024-02-07T12:45:07.562 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=xfs 2024-02-07T12:45:07.562 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=node_exporter.go:117 level=info collector=zfs 2024-02-07T12:45:07.562 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-02-07T12:45:07.562 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b[62408]: ts=2024-02-07T12:45:07.220Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-02-07T12:45:07.562 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 bash[62358]: 04ea81f79d853bc93a320860fb65f18461b68ebef6cc94f5d7a6fb2cf02f41da 2024-02-07T12:45:07.563 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:45:07 smithi172 systemd[1]: Started Ceph node-exporter.b for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:45:07.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:07 smithi019 ceph-mon[29267]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:45:07.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:07 smithi019 ceph-mon[24574]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:45:08.468 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:08 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:08.468 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:08 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:08.468 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:08 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:45:08.468 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:08 smithi019 ceph-mon[29267]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:45:08.469 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:08 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:08.469 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:08 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:08.469 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:08 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:45:08.469 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:08 smithi019 ceph-mon[24574]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:45:08.512 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:08 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:08.512 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:08 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:08.513 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:08 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:45:08.513 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:08 smithi172 ceph-mon[25457]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:45:09.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:09 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:45:09.789 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:09 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:45:09.790 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:09 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:45:10.400 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:10 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:45:10.400 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:10 smithi172 ceph-mon[25457]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:45:10.641 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:10 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:45:10.641 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:10 smithi019 ceph-mon[29267]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:45:10.641 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:10 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:45:10.641 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:10 smithi019 ceph-mon[24574]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:45:12.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:12 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:12.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:12 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:12.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:12 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:12.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:12 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:12.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:12 smithi019 ceph-mon[24574]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:45:12.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:12 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:12.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:12 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:12.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:12 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:12.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:12 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:12.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:12 smithi019 ceph-mon[29267]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:45:13.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:12 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:13.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:12 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:13.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:12 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:13.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:12 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:13.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:12 smithi172 ceph-mon[25457]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:45:14.003 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:13 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:14.004 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:13 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:14.004 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:13 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:14.004 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:13 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:14.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:45:13] "GET /metrics HTTP/1.1" 200 37988 "" "Prometheus/2.33.4" 2024-02-07T12:45:14.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:13 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:14.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:13 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:14.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:13 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:14.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:13 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:14.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:13 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:14.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:13 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:14.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:13 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:14.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:13 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:14.908 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:14 smithi019 ceph-mon[24574]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:45:14.908 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:14 smithi019 ceph-mon[29267]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:45:14.928 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:14 smithi172 ceph-mon[25457]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:45:17.526 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:17 smithi019 ceph-mon[29267]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:45:17.526 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:17 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:17.526 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:17 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:17.526 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:17 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:17.526 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:17 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:17.526 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:17 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:45:17.526 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:17 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:45:17.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:17 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:17.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:17 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:45:17.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:17 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:45:17.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:17 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:45:17.527 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:17 smithi019 ceph-mon[24574]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:45:17.527 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:17 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:17.527 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:17 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:17.527 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:17 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:17.527 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:17 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:17.527 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:17 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:45:17.527 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:17 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:45:17.528 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:17 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:17.528 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:17 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:45:17.528 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:17 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:45:17.528 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:17 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:45:17.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:17 smithi172 ceph-mon[25457]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:45:17.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:17 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:17.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:17 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:17.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:17 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:17.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:17 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:17.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:17 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:45:17.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:17 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:45:17.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:17 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:17.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:17 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:45:17.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:17 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:45:17.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:17 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:45:18.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-mon[24574]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 993 B/s rd, 0 op/s 2024-02-07T12:45:18.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-mon[24574]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:45:18.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-mon[24574]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:45:18.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-mon[24574]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:45:18.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-mon[29267]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 993 B/s rd, 0 op/s 2024-02-07T12:45:18.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-mon[29267]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:45:18.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-mon[29267]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:45:18.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-mon[29267]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:45:18.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:45:18.285+0000 7eff60d5c700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:45:18.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:45:18.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:45:18.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:45:18.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:45:18.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:18.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:18.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:45:18.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:45:18.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:45:18.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:45:18.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:45:18.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:45:18.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:45:18.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:45:18.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:45:18.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:45:18.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:45:18.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:45:18.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:18.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:18.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:45:18.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:45:18.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:45:18.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:45:18.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:45:18.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:45:18.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:45:18.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:45:18.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:45:18.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:45:18.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:45:18.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:45:18.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:45:18.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:45:18.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:45:18.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:45:18.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:45:18.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:45:18.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:45:18.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:45:18.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:45:18.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:45:18.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:18.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:18.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:45:18.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:45:18.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:45:18.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:45:18.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:45:18.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:45:18.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:45:18.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:45:18.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:45:18.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:45:18.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:45:18.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:45:18.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:18.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:18.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:18 smithi172 ceph-mon[25457]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 993 B/s rd, 0 op/s 2024-02-07T12:45:18.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:18 smithi172 ceph-mon[25457]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:45:18.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:18 smithi172 ceph-mon[25457]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:45:18.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:18 smithi172 ceph-mon[25457]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:45:19.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:19 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:45:19.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:19 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:19.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:19 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:19.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:19 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:19.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:19 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:19.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:19 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:19.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:19 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:19.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:19 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:19.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:19 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:19.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:19 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:19.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:19 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:45:19.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:19 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:19.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:19 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:19.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:19 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:19.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:19 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:19.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:19 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:19.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:19 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:19.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:19 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:19.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:19 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:19.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:19 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:19.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:19 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:45:19.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:19 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:19.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:19 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:19.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:19 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:19.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:19 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:19.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:19 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:19.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:19 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:19.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:19 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:19.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:19 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:19.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:19 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:20.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:20 smithi019 ceph-mon[24574]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:45:20.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:45:20.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:45:20.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:45:20.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:45:20.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:20.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:20.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:45:20.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:45:20.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:45:20.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:45:20.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:45:20.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:45:20.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:45:20.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:45:20.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:45:20.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:45:20.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:45:20.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:45:20.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:20.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:20.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:45:20.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:45:20.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:45:20.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:45:20.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:45:20.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:45:20.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:45:20.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:45:20.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:45:20.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:45:20.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:45:20.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:45:20.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:45:20.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:45:20.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:45:20.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:45:20.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:45:20.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:45:20.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:45:20.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:45:20.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:45:20.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:45:20.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:20.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:20.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:45:20.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:45:20.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:45:20.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:45:20.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:45:20.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:45:20.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:45:20.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:45:20.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:45:20.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:20 smithi019 ceph-mon[29267]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:45:20.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:45:20.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:45:20.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:45:20.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:45:20.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:20.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:20.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:45:20.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:45:20.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:45:20.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:45:20.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:45:20.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:45:20.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:45:20.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:45:20.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:45:20.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:45:20.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:45:20.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:45:20.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:20.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:20.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:45:20.355 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:45:20.355 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:45:20.355 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:45:20.355 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:45:20.355 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:45:20.355 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:45:20.355 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:45:20.355 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:45:20.355 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:45:20.355 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:45:20.355 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:45:20.355 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:45:20.355 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:45:20.355 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:45:20.356 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:45:20.356 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:45:20.356 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:45:20.356 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:45:20.356 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:45:20.356 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:45:20.356 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:45:20.356 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:20.356 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:20.356 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:45:20.356 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:45:20.356 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:45:20.356 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:45:20.356 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:45:20.357 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:45:20.357 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:45:20.357 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:45:20.357 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:45:20.357 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:45:20.357 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:45:20.357 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:45:20.357 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:20.357 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:20.357 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:20 smithi019 ceph-mon[24574]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 616 B/s rd, 0 op/s 2024-02-07T12:45:20.358 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:45:20.358 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:45:20.358 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:45:20.358 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:20.358 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:20.358 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:20 smithi019 ceph-mon[29267]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 616 B/s rd, 0 op/s 2024-02-07T12:45:20.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:20 smithi172 ceph-mon[25457]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:45:20.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:45:20.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:45:20.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:45:20.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:45:20.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:20.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:20.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:45:20.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:45:20.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:45:20.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:45:20.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:45:20.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:45:20.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:45:20.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:45:20.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:45:20.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:45:20.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:45:20.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:45:20.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:20.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:20.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:45:20.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:45:20.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:45:20.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:45:20.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:45:20.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:45:20.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:45:20.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:45:20.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:45:20.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:45:20.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:45:20.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:45:20.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:45:20.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:45:20.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:45:20.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:45:20.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:45:20.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:45:20.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:45:20.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:45:20.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:45:20.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:45:20.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:20.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:20.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:45:20.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:45:20.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:45:20.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:45:20.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:45:20.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:45:20.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:45:20.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:45:20.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:45:20.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:45:20.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:45:20.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:45:20.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:20.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:20.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:20 smithi172 ceph-mon[25457]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 616 B/s rd, 0 op/s 2024-02-07T12:45:21.505 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:21 smithi172 ceph-mon[25457]: Upgrade: Updating prometheus.a 2024-02-07T12:45:21.505 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:21 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:45:21.505 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:21 smithi172 ceph-mon[25457]: Deploying daemon prometheus.a on smithi172 2024-02-07T12:45:21.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:21 smithi019 ceph-mon[24574]: Upgrade: Updating prometheus.a 2024-02-07T12:45:21.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:21 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:45:21.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:21 smithi019 ceph-mon[24574]: Deploying daemon prometheus.a on smithi172 2024-02-07T12:45:21.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:21 smithi019 ceph-mon[29267]: Upgrade: Updating prometheus.a 2024-02-07T12:45:21.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:21 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:45:21.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:21 smithi019 ceph-mon[29267]: Deploying daemon prometheus.a on smithi172 2024-02-07T12:45:22.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:22 smithi172 ceph-mon[25457]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:45:22.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:22 smithi019 ceph-mon[24574]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:45:22.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:22 smithi019 ceph-mon[29267]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:45:23.871 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:23 smithi172 systemd[1]: Stopping Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:45:23.871 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:23 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:45:23.633Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-02-07T12:45:23.871 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:23 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:45:23.633Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-02-07T12:45:23.871 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:23 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:45:23.633Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-02-07T12:45:23.871 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:23 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:45:23.633Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-02-07T12:45:23.871 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:23 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:45:23.633Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-02-07T12:45:23.871 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:23 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:45:23.633Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-02-07T12:45:23.871 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:23 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:45:23.633Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-02-07T12:45:23.872 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:23 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:45:23.633Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-02-07T12:45:23.872 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:23 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:45:23.634Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-02-07T12:45:23.872 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:23 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:45:23.634Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-02-07T12:45:23.872 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:23 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:45:23.634Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-02-07T12:45:23.872 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:23 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[60332]: ts=2024-02-07T12:45:23.634Z caller=main.go:1066 level=info msg="See you next time!" 2024-02-07T12:45:23.872 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:23 smithi172 podman[63769]: 2024-02-07 12:45:23.643793936 +0000 UTC m=+0.104208251 container died 80e3dbe199442a6bda4f535b03f55ecf18b609ab228f1269f45a0c29c9da3333 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:45:24.254 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:23 smithi172 podman[63769]: 2024-02-07 12:45:23.991633145 +0000 UTC m=+0.452047459 container cleanup 80e3dbe199442a6bda4f535b03f55ecf18b609ab228f1269f45a0c29c9da3333 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:45:24.254 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:23 smithi172 bash[63769]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a 2024-02-07T12:45:24.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:24 smithi019 ceph-mon[24574]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:45:24.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:24 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:45:24.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:24 smithi019 ceph-mon[29267]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:45:24.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:24 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:45:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:24 smithi172 ceph-mon[25457]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:45:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:24 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:45:24.619 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:24 smithi172 podman[63784]: 2024-02-07 12:45:24.443371464 +0000 UTC m=+0.797671680 container remove 80e3dbe199442a6bda4f535b03f55ecf18b609ab228f1269f45a0c29c9da3333 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:45:25.069 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:24 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@prometheus.a.service: Deactivated successfully. 2024-02-07T12:45:25.069 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:24 smithi172 systemd[1]: Stopped Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:45:25.069 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:24 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@prometheus.a.service: Consumed 1.249s CPU time. 2024-02-07T12:45:25.069 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:24 smithi172 systemd[1]: Starting Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:45:25.332 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 podman[63917]: 2024-02-07 12:45:24.97133958 +0000 UTC m=+0.019923995 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-02-07T12:45:25.332 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 podman[63917]: 2024-02-07 12:45:25.094607744 +0000 UTC m=+0.143192199 container create c9b607eea1be415ba6cacefac1bbc72efcc7c587f1ca609bd355be1bcca50d55 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:45:25.332 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 podman[63917]: 2024-02-07 12:45:25.304837738 +0000 UTC m=+0.353422154 container init c9b607eea1be415ba6cacefac1bbc72efcc7c587f1ca609bd355be1bcca50d55 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:45:25.332 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 podman[63917]: 2024-02-07 12:45:25.30735723 +0000 UTC m=+0.355941646 container start c9b607eea1be415ba6cacefac1bbc72efcc7c587f1ca609bd355be1bcca50d55 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:45:25.619 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:45:25.330Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-02-07T12:45:25.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:45:25.330Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-02-07T12:45:25.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:45:25.330Z caller=main.go:570 level=info host_details="(Linux 5.14.0-418.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Mon Feb 5 23:17:48 UTC 2024 x86_64 smithi172 (none))" 2024-02-07T12:45:25.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:45:25.330Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-02-07T12:45:25.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:45:25.330Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-02-07T12:45:25.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:45:25.332Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-02-07T12:45:25.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:45:25.332Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-02-07T12:45:25.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:45:25.333Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-02-07T12:45:25.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:45:25.333Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-02-07T12:45:25.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:45:25.335Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-02-07T12:45:25.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:45:25.335Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.389µs 2024-02-07T12:45:25.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:45:25.335Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-02-07T12:45:25.620 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:45:25.342Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=6 2024-02-07T12:45:25.621 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:45:25.356Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=6 2024-02-07T12:45:25.621 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:45:25.364Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=6 2024-02-07T12:45:25.621 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:45:25.367Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=6 2024-02-07T12:45:25.621 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:45:25.369Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=6 2024-02-07T12:45:25.621 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:45:25.370Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=6 2024-02-07T12:45:25.621 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:45:25.370Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=6 maxSegment=6 2024-02-07T12:45:25.621 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:45:25.370Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=24.138µs wal_replay_duration=35.480047ms wbl_replay_duration=111ns total_replay_duration=35.521926ms 2024-02-07T12:45:25.621 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:45:25.373Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-02-07T12:45:25.621 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:45:25.373Z caller=main.go:1029 level=info msg="TSDB started" 2024-02-07T12:45:25.621 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:45:25.373Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-02-07T12:45:25.621 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:45:25.392Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=18.779658ms db_storage=841ns remote_storage=1.096µs web_handler=340ns query_engine=618ns scrape=255.549µs scrape_sd=91.456µs notify=15.704µs notify_sd=15.65µs rules=18.186802ms tracing=6.005µs 2024-02-07T12:45:25.621 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:45:25.392Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-02-07T12:45:25.621 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:45:25.392Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-02-07T12:45:25.621 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 bash[63917]: c9b607eea1be415ba6cacefac1bbc72efcc7c587f1ca609bd355be1bcca50d55 2024-02-07T12:45:25.622 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:45:25 smithi172 systemd[1]: Started Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:45:26.544 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:26 smithi172 ceph-mon[25457]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 616 B/s rd, 0 op/s 2024-02-07T12:45:26.544 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:26 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:26.544 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:26 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:26.544 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:26 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:45:26.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:26 smithi019 ceph-mon[24574]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 616 B/s rd, 0 op/s 2024-02-07T12:45:26.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:26 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:26.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:26 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:26.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:26 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:45:26.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:26 smithi019 ceph-mon[29267]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 616 B/s rd, 0 op/s 2024-02-07T12:45:26.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:26 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:26.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:26 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:26.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:26 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:45:27.972 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:45:28.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:28 smithi019 ceph-mon[24574]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:45:28.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:28 smithi019 ceph-mon[29267]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:45:28.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:28 smithi172 ceph-mon[25457]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:45:28.638 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:45:28.638 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (13m) 16s ago 21m 23.8M - 0.23.0 ba2b418f427c dfaf78b29ab5 2024-02-07T12:45:28.638 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (13m) 16s ago 21m 52.7M - 8.3.5 dad864ee21e9 e753e96fa4f0 2024-02-07T12:45:28.638 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 unknown 16s ago 20m 57.2M - 3.5 e1d6a67b021e 5113aa77e3f2 2024-02-07T12:45:28.638 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283,8765 running (46s) 16s ago 26m 450M - 19.0.0-1252-g44458cce 79957b372307 5aa0c04fdfd7 2024-02-07T12:45:28.638 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (10m) 16s ago 27m 499M - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:45:28.639 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (27m) 16s ago 27m 78.4M 2048M 17.2.0 e1d6a67b021e f3a8a050cf1d 2024-02-07T12:45:28.639 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (26m) 16s ago 26m 57.7M 2048M 17.2.0 e1d6a67b021e 7d62c7c4d075 2024-02-07T12:45:28.639 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (26m) 16s ago 26m 60.6M 2048M 17.2.0 e1d6a67b021e b9825d0db32d 2024-02-07T12:45:28.639 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (28s) 16s ago 22m 7952k - 1.5.0 0da6a335fe13 c4bdcb1c4d49 2024-02-07T12:45:28.639 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (21s) 16s ago 22m 3409k - 1.5.0 0da6a335fe13 04ea81f79d85 2024-02-07T12:45:28.639 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (25m) 16s ago 25m 54.4M 2993M 17.2.0 e1d6a67b021e 87f2560a9b15 2024-02-07T12:45:28.639 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (25m) 16s ago 25m 57.0M 2993M 17.2.0 e1d6a67b021e 0f9a190f0e0d 2024-02-07T12:45:28.639 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (25m) 16s ago 24m 52.7M 2993M 17.2.0 e1d6a67b021e 7db94f25a1aa 2024-02-07T12:45:28.639 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (24m) 16s ago 24m 53.2M 2993M 17.2.0 e1d6a67b021e 0b6b674188ed 2024-02-07T12:45:28.639 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (24m) 16s ago 24m 55.2M 3248M 17.2.0 e1d6a67b021e df72cfd69b3b 2024-02-07T12:45:28.639 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (23m) 16s ago 23m 52.5M 3248M 17.2.0 e1d6a67b021e 3bdb22c07537 2024-02-07T12:45:28.639 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (23m) 16s ago 23m 52.0M 3248M 17.2.0 e1d6a67b021e 7859b651e0d9 2024-02-07T12:45:28.639 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (23m) 16s ago 23m 55.1M 3248M 17.2.0 e1d6a67b021e b301e7c75d73 2024-02-07T12:45:28.639 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 starting - - - - 2024-02-07T12:45:28.640 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (20m) 16s ago 20m 77.7M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:45:28.640 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (20m) 16s ago 20m 77.2M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:45:29.038 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:45:29.039 INFO:teuthology.orchestra.run.smithi019.stdout: "mon": { 2024-02-07T12:45:29.039 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-02-07T12:45:29.039 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:45:29.039 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr": { 2024-02-07T12:45:29.039 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:45:29.039 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:45:29.039 INFO:teuthology.orchestra.run.smithi019.stdout: "osd": { 2024-02-07T12:45:29.039 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-02-07T12:45:29.039 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:45:29.039 INFO:teuthology.orchestra.run.smithi019.stdout: "mds": {}, 2024-02-07T12:45:29.039 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw": { 2024-02-07T12:45:29.039 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:45:29.040 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:45:29.040 INFO:teuthology.orchestra.run.smithi019.stdout: "overall": { 2024-02-07T12:45:29.040 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-02-07T12:45:29.040 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:45:29.040 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:45:29.040 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:45:29.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:29 smithi019 ceph-mon[29267]: from='client.15435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:45:29.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:29 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1214574716' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:29.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:29 smithi019 ceph-mon[24574]: from='client.15435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:45:29.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:29 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1214574716' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:29.378 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:45:29.378 INFO:teuthology.orchestra.run.smithi019.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:45:29.378 INFO:teuthology.orchestra.run.smithi019.stdout: "in_progress": true, 2024-02-07T12:45:29.378 INFO:teuthology.orchestra.run.smithi019.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-02-07T12:45:29.378 INFO:teuthology.orchestra.run.smithi019.stdout: "services_complete": [ 2024-02-07T12:45:29.378 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr" 2024-02-07T12:45:29.378 INFO:teuthology.orchestra.run.smithi019.stdout: ], 2024-02-07T12:45:29.378 INFO:teuthology.orchestra.run.smithi019.stdout: "progress": "2/2 daemons upgraded", 2024-02-07T12:45:29.379 INFO:teuthology.orchestra.run.smithi019.stdout: "message": "Currently upgrading prometheus daemons", 2024-02-07T12:45:29.379 INFO:teuthology.orchestra.run.smithi019.stdout: "is_paused": false 2024-02-07T12:45:29.379 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:45:29.565 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:29 smithi172 ceph-mon[25457]: from='client.15435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:45:29.565 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:29 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/1214574716' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:30.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:30 smithi019 ceph-mon[29267]: from='client.25327 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:45:30.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:30 smithi019 ceph-mon[29267]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:45:30.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:30 smithi019 ceph-mon[29267]: from='client.25333 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:45:30.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:30 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:30.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:30 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:30.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:30 smithi019 ceph-mon[24574]: from='client.25327 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:45:30.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:30 smithi019 ceph-mon[24574]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:45:30.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:30 smithi019 ceph-mon[24574]: from='client.25333 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:45:30.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:30 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:30.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:30 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:30.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:30 smithi172 ceph-mon[25457]: from='client.25327 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:45:30.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:30 smithi172 ceph-mon[25457]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:45:30.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:30 smithi172 ceph-mon[25457]: from='client.25333 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:45:30.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:30 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:30.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:30 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:31.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:31 smithi019 ceph-mon[24574]: from='client.15459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:45:31.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:31 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:45:31.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:31 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:31.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:31 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:31.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:31 smithi019 ceph-mon[29267]: from='client.15459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:45:31.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:31 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:45:31.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:31 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:31.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:31 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:31.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:31 smithi172 ceph-mon[25457]: from='client.15459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:45:31.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:31 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:45:31.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:31 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:31.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:31 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:32.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:32 smithi019 ceph-mon[29267]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:45:32.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:32 smithi019 ceph-mon[24574]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:45:32.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:32 smithi172 ceph-mon[25457]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:45:34.218 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:45:33] "GET /metrics HTTP/1.1" 200 37983 "" "Prometheus/2.43.0" 2024-02-07T12:45:34.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:34 smithi019 ceph-mon[24574]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:45:34.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:34 smithi019 ceph-mon[29267]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:45:34.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:34 smithi172 ceph-mon[25457]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:45:35.315 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:35 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:35.315 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:35 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:35.315 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:35 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:45:35.315 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:35 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:45:35.316 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:35 smithi019 ceph-mon[24574]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 868 B/s rd, 0 op/s 2024-02-07T12:45:35.316 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:35 smithi019 ceph-mon[24574]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:45:35.316 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:35 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:35.316 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:35 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:45:35.316 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:35 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:45:35.316 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:35 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:45:35.316 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:35 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:35.316 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:35 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:35.316 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:35 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:45:35.316 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:35 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:45:35.317 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:35 smithi019 ceph-mon[29267]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 868 B/s rd, 0 op/s 2024-02-07T12:45:35.317 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:35 smithi019 ceph-mon[29267]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:45:35.317 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:35 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:35.317 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:35 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:45:35.317 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:35 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:45:35.317 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:35 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:45:35.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:35 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:35.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:35 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:35.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:35 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:45:35.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:35 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:45:35.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:35 smithi172 ceph-mon[25457]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 868 B/s rd, 0 op/s 2024-02-07T12:45:35.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:35 smithi172 ceph-mon[25457]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:45:35.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:35 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:35.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:35 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:45:35.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:35 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:45:35.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:35 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:45:36.267 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:45:36.125+0000 7eff60d5c700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:45:36.267 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:45:36.267 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:45:36.267 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:45:36.267 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:45:36.267 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:36.268 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:36.268 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:45:36.268 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:45:36.268 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:45:36.268 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:45:36.268 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:45:36.268 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:45:36.268 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:45:36.268 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:45:36.268 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:45:36.268 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:45:36.268 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:45:36.269 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:45:36.269 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:36.269 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:36.269 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:45:36.269 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:45:36.269 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:45:36.269 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:45:36.270 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:45:36.270 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:45:36.270 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:45:36.270 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:45:36.270 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:45:36.270 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:45:36.270 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:45:36.270 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:45:36.270 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:45:36.270 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:45:36.270 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:45:36.270 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:45:36.271 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:45:36.271 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:45:36.271 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:45:36.271 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:45:36.271 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:45:36.271 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:45:36.271 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:36.271 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[29267]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:45:36.271 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[29267]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:45:36.271 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:45:36.271 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:45:36.272 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.272 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.272 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.272 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.272 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.272 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.272 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.272 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.272 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.272 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.273 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:36.273 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:45:36.273 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:45:36.273 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:45:36.273 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:45:36.273 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:45:36.273 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:45:36.273 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:45:36.273 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:45:36.273 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:45:36.273 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:45:36.274 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:45:36.274 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:45:36.274 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:36.274 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:36.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[24574]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:45:36.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[24574]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:45:36.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:45:36.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:45:36.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:36 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:36 smithi172 ceph-mon[25457]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:45:36.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:36 smithi172 ceph-mon[25457]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:45:36.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:36 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:45:36.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:36 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:45:36.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:36 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:36 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:36 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:36 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:36 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:36 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:36 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:36 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:36 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:36.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:36 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:37.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:37 smithi019 ceph-mon[24574]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:45:37.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:45:37.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:45:37.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:45:37.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:45:37.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:37.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:37.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:45:37.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:45:37.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:45:37.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:45:37.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:45:37.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:45:37.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:45:37.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:45:37.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:45:37.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:45:37.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:45:37.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:45:37.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:37.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:37.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:45:37.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:45:37.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:45:37.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:45:37.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:45:37.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:45:37.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:45:37.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:45:37.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:45:37.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:45:37.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:45:37.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:45:37.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:45:37.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:45:37.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:45:37.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:45:37.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:45:37.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:45:37.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:45:37.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:45:37.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:45:37.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:45:37.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:37.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:37.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:45:37.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:45:37.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:45:37.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:45:37.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:45:37.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:45:37.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:45:37.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:45:37.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:45:37.599 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:37 smithi019 ceph-mon[29267]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:45:37.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:45:37.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:45:37.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:45:37.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:45:37.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:37.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:37.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:45:37.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:45:37.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:45:37.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:45:37.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:45:37.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:45:37.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:45:37.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:45:37.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:45:37.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:45:37.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:45:37.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:45:37.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:37.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:37.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:45:37.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:45:37.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:45:37.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:45:37.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:45:37.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:45:37.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:45:37.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:45:37.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:45:37.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:45:37.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:45:37.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:45:37.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:45:37.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:45:37.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:45:37.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:45:37.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:45:37.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:45:37.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:45:37.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:45:37.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:45:37.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:45:37.605 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:37.605 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:37.605 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:45:37.605 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:45:37.605 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:45:37.605 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:45:37.605 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:45:37.605 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:45:37.605 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:45:37.605 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:45:37.605 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:45:37.606 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:45:37.606 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:45:37.606 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:45:37.606 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:37.606 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:37.606 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:37 smithi019 ceph-mon[24574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:45:37.606 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:37 smithi019 ceph-mon[24574]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:45:37.606 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:45:37.606 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:45:37.607 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:45:37.607 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:37.607 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:37.607 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:37 smithi019 ceph-mon[29267]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:45:37.607 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:37 smithi019 ceph-mon[29267]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:45:37.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:37 smithi172 ceph-mon[25457]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:45:37.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:45:37.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:45:37.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:45:37.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:45:37.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:37.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:37.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:45:37.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:45:37.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:45:37.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:45:37.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:45:37.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:45:37.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:45:37.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:45:37.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:45:37.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:45:37.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:45:37.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:45:37.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:37.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:37.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:45:37.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:45:37.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:45:37.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:45:37.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:45:37.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:45:37.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:45:37.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:45:37.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:45:37.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:45:37.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:45:37.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:45:37.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:45:37.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:45:37.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:45:37.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:45:37.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:45:37.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:45:37.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:45:37.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:45:37.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:45:37.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:45:37.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:37.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:37.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:45:37.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:45:37.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:45:37.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:45:37.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:45:37.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:45:37.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:45:37.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:45:37.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:45:37.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:45:37.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:45:37.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:45:37.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:37.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:37.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:37 smithi172 ceph-mon[25457]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:45:37.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:37 smithi172 ceph-mon[25457]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:45:39.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:39 smithi019 ceph-mon[24574]: Upgrade: Updating alertmanager.a 2024-02-07T12:45:39.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:39 smithi019 ceph-mon[24574]: Deploying daemon alertmanager.a on smithi019 2024-02-07T12:45:39.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:39 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:45:39.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:39 smithi019 ceph-mon[29267]: Upgrade: Updating alertmanager.a 2024-02-07T12:45:39.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:39 smithi019 ceph-mon[29267]: Deploying daemon alertmanager.a on smithi019 2024-02-07T12:45:39.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:39 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:45:39.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:39 smithi172 ceph-mon[25457]: Upgrade: Updating alertmanager.a 2024-02-07T12:45:39.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:39 smithi172 ceph-mon[25457]: Deploying daemon alertmanager.a on smithi019 2024-02-07T12:45:39.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:39 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:45:40.494 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:40 smithi019 ceph-mon[24574]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:45:40.494 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:40 smithi019 ceph-mon[29267]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:45:40.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:40 smithi172 ceph-mon[25457]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:45:41.439 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:41 smithi019 systemd[1]: Stopping Ceph alertmanager.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:45:41.440 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[58688]: level=info ts=2024-02-07T12:45:41.396Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-02-07T12:45:41.440 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:41 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:45:41.440 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:41 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:45:41.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:41 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:45:41.732 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:41 smithi019 podman[76395]: 2024-02-07 12:45:41.437844712 +0000 UTC m=+0.141576917 container died dfaf78b29ab547df327fe28e962aae788b67a6c28cecc0e979259756a2f4ceaf (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-02-07T12:45:42.041 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:41 smithi019 podman[76395]: 2024-02-07 12:45:41.822014607 +0000 UTC m=+0.525746810 container cleanup dfaf78b29ab547df327fe28e962aae788b67a6c28cecc0e979259756a2f4ceaf (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-02-07T12:45:42.041 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:41 smithi019 bash[76395]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a 2024-02-07T12:45:42.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:42 smithi019 ceph-mon[29267]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:45:42.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:42 smithi019 ceph-mon[24574]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:45:42.345 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:42 smithi019 podman[76410]: 2024-02-07 12:45:42.040082561 +0000 UTC m=+0.593727259 container remove dfaf78b29ab547df327fe28e962aae788b67a6c28cecc0e979259756a2f4ceaf (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-02-07T12:45:42.345 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:42 smithi019 podman[76410]: 2024-02-07 12:45:42.115876072 +0000 UTC m=+0.669520769 volume remove 63980714c49c66f0d315df879326979f134ed4e9bff9ed0429813c07a76daca1 2024-02-07T12:45:42.345 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:42 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@alertmanager.a.service: Deactivated successfully. 2024-02-07T12:45:42.345 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:42 smithi019 systemd[1]: Stopped Ceph alertmanager.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:45:42.345 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:42 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@alertmanager.a.service: Consumed 1.460s CPU time. 2024-02-07T12:45:42.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:42 smithi172 ceph-mon[25457]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:45:42.793 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:42 smithi019 systemd[1]: Starting Ceph alertmanager.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:45:42.794 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:42 smithi019 podman[76543]: 2024-02-07 12:45:42.56534101 +0000 UTC m=+0.017593400 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-02-07T12:45:42.794 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:42 smithi019 podman[76543]: 2024-02-07 12:45:42.683618003 +0000 UTC m=+0.135870425 volume create bd14446a59981c254226057a9ae8a5b50255d6a115e3608cbb998b9abd1c9242 2024-02-07T12:45:43.071 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:42 smithi019 podman[76543]: 2024-02-07 12:45:42.792381182 +0000 UTC m=+0.244633601 container create fec9255f35e7f325c423bf0f247b5f327b09b25c9bdffc61bb052ef40a0052bb (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-02-07T12:45:43.344 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:43 smithi019 podman[76543]: 2024-02-07 12:45:43.06995804 +0000 UTC m=+0.522210431 container init fec9255f35e7f325c423bf0f247b5f327b09b25c9bdffc61bb052ef40a0052bb (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-02-07T12:45:43.344 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:43 smithi019 podman[76543]: 2024-02-07 12:45:43.072815302 +0000 UTC m=+0.525067687 container start fec9255f35e7f325c423bf0f247b5f327b09b25c9bdffc61bb052ef40a0052bb (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-02-07T12:45:43.344 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:45:43.084Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-02-07T12:45:43.345 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:45:43.085Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-02-07T12:45:43.345 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:45:43.086Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.19 port=9094 2024-02-07T12:45:43.345 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:45:43.088Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-02-07T12:45:43.345 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:45:43.111Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-02-07T12:45:43.345 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:45:43.111Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-02-07T12:45:43.345 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:45:43.114Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-02-07T12:45:43.345 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:45:43.114Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-02-07T12:45:43.345 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:43 smithi019 bash[76543]: fec9255f35e7f325c423bf0f247b5f327b09b25c9bdffc61bb052ef40a0052bb 2024-02-07T12:45:43.345 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:43 smithi019 systemd[1]: Started Ceph alertmanager.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:45:44.232 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:45:43] "GET /metrics HTTP/1.1" 200 37983 "" "Prometheus/2.43.0" 2024-02-07T12:45:44.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:44 smithi172 ceph-mon[25457]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:45:44.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:44 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:44.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:44 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:44.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:44 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:45:44.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:44 smithi019 ceph-mon[24574]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:45:44.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:44 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:44.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:44 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:44.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:44 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:45:44.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:44 smithi019 ceph-mon[29267]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:45:44.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:44 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:44.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:44 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:44.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:44 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:45:45.344 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:45 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:45:45.088Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000421235s 2024-02-07T12:45:46.367 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:46 smithi019 ceph-mon[24574]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:45:46.367 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:46 smithi019 ceph-mon[29267]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:45:46.602 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:46 smithi172 ceph-mon[25457]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:45:48.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:48 smithi019 ceph-mon[29267]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:45:48.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:48 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:48.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:48 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:48.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:48 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:48.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:48 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:48.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:48 smithi019 ceph-mon[24574]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:45:48.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:48 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:48.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:48 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:48.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:48 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:48.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:48 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:48.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:48 smithi172 ceph-mon[25457]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:45:48.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:48 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:48.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:48 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:48.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:48 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:48.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:48 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:49.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:49 smithi019 ceph-mon[29267]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:45:49.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:49 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:49.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:49 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:49.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:49 smithi019 ceph-mon[24574]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:45:49.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:49 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:49.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:49 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:49.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:49 smithi172 ceph-mon[25457]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:45:49.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:49 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:49.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:49 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:50.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:50 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:45:50.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:50 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:45:50.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:50 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:45:51.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:51 smithi019 ceph-mon[24574]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:45:51.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:51 smithi019 ceph-mon[29267]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:45:51.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:51 smithi172 ceph-mon[25457]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:45:53.158 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:45:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:45:53.090Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002132433s 2024-02-07T12:45:53.158 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:53 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:53.158 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:53 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:53.158 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:53 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:45:53.158 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:53 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:45:53.159 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:53 smithi019 ceph-mon[24574]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 0 op/s 2024-02-07T12:45:53.159 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:53 smithi019 ceph-mon[24574]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:45:53.159 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:53 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:53.159 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:53 smithi019 ceph-mon[24574]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:45:53.159 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:53 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:45:53.159 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:53 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:45:53.159 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:53 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:45:53.159 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:53 smithi019 ceph-mon[24574]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:45:53.159 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:53 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:53.159 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:53 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:53.159 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:53 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:45:53.160 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:53 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:45:53.160 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:53 smithi019 ceph-mon[29267]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 0 op/s 2024-02-07T12:45:53.160 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:53 smithi019 ceph-mon[29267]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:45:53.160 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:53 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:53.160 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:53 smithi019 ceph-mon[29267]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:45:53.160 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:53 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:45:53.160 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:53 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:45:53.160 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:53 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:45:53.160 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:53 smithi019 ceph-mon[29267]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:45:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:53 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:53 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:53 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:45:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:53 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:45:53.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:53 smithi172 ceph-mon[25457]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 0 op/s 2024-02-07T12:45:53.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:53 smithi172 ceph-mon[25457]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:45:53.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:53 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:45:53.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:53 smithi172 ceph-mon[25457]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:45:53.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:53 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:45:53.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:53 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:45:53.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:53 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:45:53.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:53 smithi172 ceph-mon[25457]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:45:54.227 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:45:53] "GET /metrics HTTP/1.1" 200 37983 "" "Prometheus/2.43.0" 2024-02-07T12:45:54.227 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:45:54.082+0000 7eff60d5c700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:45:54.227 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:45:54.227 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:45:54.227 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:45:54.227 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:45:54.227 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:54.227 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:54.227 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:45:54.227 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:45:54.227 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:45:54.228 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:45:54.228 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:45:54.228 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:45:54.228 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:45:54.228 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:45:54.228 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:45:54.228 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:45:54.228 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:45:54.229 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:45:54.229 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:54.229 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:54.229 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:45:54.229 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:45:54.229 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:45:54.229 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:45:54.229 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:45:54.229 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:45:54.229 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:45:54.229 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:45:54.229 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:45:54.230 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:45:54.230 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:45:54.230 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:45:54.230 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:45:54.230 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:45:54.230 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:45:54.230 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:45:54.230 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:45:54.230 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:45:54.230 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:45:54.230 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:45:54.230 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:45:54.230 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:45:54.230 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:54.230 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:54.231 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:45:54.231 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:45:54.231 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:45:54.231 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:45:54.231 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:45:54.231 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:45:54.231 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:45:54.231 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:45:54.231 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:45:54.231 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:45:54.231 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:45:54.231 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:45:54.231 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:54.231 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:54.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:45:54.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:45:54.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:45:54.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:45:54.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:45:54.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:45:54.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:54 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:54 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:45:54.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:54 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:45:54.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:54 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:45:54.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:54 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:54 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:54 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:54 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:54 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:54 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:54 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:54 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:54 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:54 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:54.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:54 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:45:55.580 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:55 smithi172 ceph-mon[25457]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:45:55.580 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:45:55.581 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:45:55.581 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:45:55.581 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:45:55.581 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:55.581 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:55.581 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:45:55.581 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:45:55.581 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:45:55.581 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:45:55.581 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:45:55.581 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:45:55.581 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:45:55.581 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:45:55.582 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:45:55.582 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:45:55.582 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:45:55.582 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:45:55.582 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:55.582 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:55.582 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:45:55.582 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:45:55.582 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:45:55.582 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:45:55.582 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:45:55.582 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:45:55.582 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:45:55.582 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:45:55.582 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:45:55.583 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:45:55.583 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:45:55.583 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:45:55.583 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:45:55.583 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:45:55.583 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:45:55.583 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:45:55.583 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:45:55.583 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:45:55.583 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:45:55.583 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:45:55.583 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:45:55.583 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:45:55.583 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:55.583 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:55.583 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:45:55.584 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:45:55.584 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:45:55.584 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:45:55.584 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:45:55.584 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:45:55.584 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:45:55.584 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:45:55.584 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:45:55.584 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:45:55.584 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:45:55.584 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:45:55.584 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:55.584 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:55.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:55 smithi172 ceph-mon[25457]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:45:55.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:55 smithi172 ceph-mon[25457]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:45:55.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:55 smithi019 ceph-mon[24574]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:45:55.594 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:45:55.594 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:45:55.594 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:45:55.594 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:45:55.594 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:55.594 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:55.594 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:45:55.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:45:55.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:45:55.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:45:55.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:45:55.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:45:55.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:45:55.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:45:55.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:45:55.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:45:55.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:45:55.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:45:55.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:55.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:55.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:45:55.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:45:55.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:45:55.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:45:55.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:45:55.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:45:55.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:45:55.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:45:55.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:45:55.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:45:55.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:45:55.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:45:55.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:45:55.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:45:55.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:45:55.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:45:55.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:45:55.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:45:55.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:45:55.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:45:55.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:45:55.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:45:55.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:55.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:55.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:45:55.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:45:55.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:45:55.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:45:55.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:45:55.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:45:55.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:45:55.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:45:55.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:45:55.599 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:55 smithi019 ceph-mon[29267]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:45:55.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:45:55.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:45:55.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:45:55.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:45:55.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:55.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:55.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:45:55.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:45:55.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:45:55.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:45:55.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:45:55.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:45:55.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:45:55.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:45:55.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:45:55.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:45:55.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:45:55.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:45:55.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:55.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:55.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:45:55.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:45:55.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:45:55.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:45:55.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:45:55.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:45:55.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:45:55.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:45:55.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:45:55.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:45:55.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:45:55.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:45:55.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:45:55.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:45:55.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:45:55.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:45:55.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:45:55.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:45:55.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:45:55.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:45:55.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:45:55.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:45:55.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:55.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:55.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:45:55.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:45:55.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:45:55.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:45:55.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:45:55.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:45:55.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:45:55.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:45:55.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:45:55.603 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:45:55.603 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:45:55.603 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:45:55.603 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:55.603 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:55.603 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:55 smithi019 ceph-mon[24574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:45:55.603 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:55 smithi019 ceph-mon[24574]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:45:55.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:45:55.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:45:55.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:45:55.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:45:55.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:45:55.604 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:55 smithi019 ceph-mon[29267]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:45:55.604 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:55 smithi019 ceph-mon[29267]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:45:57.388 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:57 smithi172 ceph-mon[25457]: Upgrade: Updating grafana.a 2024-02-07T12:45:57.388 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:57 smithi172 ceph-mon[25457]: Deploying daemon grafana.a on smithi172 2024-02-07T12:45:57.388 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:57 smithi172 ceph-mon[25457]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:45:57.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:57 smithi019 ceph-mon[24574]: Upgrade: Updating grafana.a 2024-02-07T12:45:57.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:57 smithi019 ceph-mon[24574]: Deploying daemon grafana.a on smithi172 2024-02-07T12:45:57.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:57 smithi019 ceph-mon[24574]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:45:57.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:57 smithi019 ceph-mon[29267]: Upgrade: Updating grafana.a 2024-02-07T12:45:57.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:57 smithi019 ceph-mon[29267]: Deploying daemon grafana.a on smithi172 2024-02-07T12:45:57.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:57 smithi019 ceph-mon[29267]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:45:58.884 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:45:58 smithi172 systemd[1]: Stopping Ceph grafana.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:45:58.884 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:45:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[49476]: t=2024-02-07T12:45:58+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-02-07T12:45:59.214 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:45:58 smithi172 podman[65872]: 2024-02-07 12:45:58.882196983 +0000 UTC m=+0.114109362 container died e753e96fa4f0cabe3a55514250eb56c433a6c35205ae213fe8076ee082a259b4 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, build-date=2022-03-28T10:36:18.413762, vendor=Red Hat, Inc., io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , description=Ceph Grafana Container, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, release=236.1648460182, vcs-type=git, com.redhat.component=ubi8-container, io.buildah.version=1.24.2, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-02-07T12:45:59.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:45:59 smithi019 ceph-mon[24574]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:45:59.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:45:59 smithi019 ceph-mon[29267]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:45:59.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:45:59 smithi172 ceph-mon[25457]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:45:59.619 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:45:59 smithi172 podman[65872]: 2024-02-07 12:45:59.212941354 +0000 UTC m=+0.444853733 container cleanup e753e96fa4f0cabe3a55514250eb56c433a6c35205ae213fe8076ee082a259b4 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, build-date=2022-03-28T10:36:18.413762, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, release=236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, version=8.5, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, io.buildah.version=1.24.2, name=ubi8) 2024-02-07T12:45:59.619 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:45:59 smithi172 bash[65872]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a 2024-02-07T12:45:59.619 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:45:59 smithi172 podman[65886]: 2024-02-07 12:45:59.456534228 +0000 UTC m=+0.572928339 container remove e753e96fa4f0cabe3a55514250eb56c433a6c35205ae213fe8076ee082a259b4 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=236.1648460182, io.openshift.tags=base rhel8, name=ubi8, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, version=8.5, description=Ceph Grafana Container, io.buildah.version=1.24.2, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-03-28T10:36:18.413762, architecture=x86_64) 2024-02-07T12:45:59.745 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:46:00.103 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:45:59 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@grafana.a.service: Deactivated successfully. 2024-02-07T12:46:00.103 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:45:59 smithi172 systemd[1]: Stopped Ceph grafana.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:46:00.103 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:45:59 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@grafana.a.service: Consumed 5.391s CPU time. 2024-02-07T12:46:00.103 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:45:59 smithi172 systemd[1]: Starting Ceph grafana.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:46:00.369 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 podman[66015]: 2024-02-07 12:46:00.004038391 +0000 UTC m=+0.017953624 image pull a0ea5e3c46d9434bd173d585b7c2638c7451eaf2543d1d22f9e78442615aaffb quay.io/ceph/ceph-grafana:9.4.12 2024-02-07T12:46:00.370 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 podman[66015]: 2024-02-07 12:46:00.107375445 +0000 UTC m=+0.121290715 container create aaffac63bc281628aa7d0fce29262aa3651c7c60651205efa0d467922a09b43c (image=quay.io/ceph/ceph-grafana:9.4.12, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, release=239.1651231664, distribution-scope=public, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, maintainer=Paul Cuzner , architecture=x86_64, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0) 2024-02-07T12:46:00.370 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 podman[66015]: 2024-02-07 12:46:00.318293753 +0000 UTC m=+0.332208981 container init aaffac63bc281628aa7d0fce29262aa3651c7c60651205efa0d467922a09b43c (image=quay.io/ceph/ceph-grafana:9.4.12, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, io.openshift.expose-services=, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, name=ubi8, io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, release=239.1651231664, maintainer=Paul Cuzner , version=8.5, com.redhat.component=ubi8-container, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-02-07T12:46:00.370 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 podman[66015]: 2024-02-07 12:46:00.321275015 +0000 UTC m=+0.335190247 container start aaffac63bc281628aa7d0fce29262aa3651c7c60651205efa0d467922a09b43c (image=quay.io/ceph/ceph-grafana:9.4.12, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, name=ubi8, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., release=239.1651231664, distribution-scope=public, io.buildah.version=1.32.0, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, vcs-type=git) 2024-02-07T12:46:00.407 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:46:00.408 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (17s) 12s ago 22m 17.3M - 0.25.0 c8568f914cd2 fec9255f35e7 2024-02-07T12:46:00.408 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (13m) 12s ago 21m 53.0M - 8.3.5 dad864ee21e9 e753e96fa4f0 2024-02-07T12:46:00.408 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 unknown 12s ago 21m 57.2M - 3.5 e1d6a67b021e 5113aa77e3f2 2024-02-07T12:46:00.408 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283,8765 running (77s) 12s ago 26m 451M - 19.0.0-1252-g44458cce 79957b372307 5aa0c04fdfd7 2024-02-07T12:46:00.408 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (11m) 12s ago 28m 503M - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:46:00.408 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (28m) 12s ago 28m 79.6M 2048M 17.2.0 e1d6a67b021e f3a8a050cf1d 2024-02-07T12:46:00.408 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (26m) 12s ago 26m 58.3M 2048M 17.2.0 e1d6a67b021e 7d62c7c4d075 2024-02-07T12:46:00.408 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (27m) 12s ago 27m 61.4M 2048M 17.2.0 e1d6a67b021e b9825d0db32d 2024-02-07T12:46:00.408 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (60s) 12s ago 22m 8632k - 1.5.0 0da6a335fe13 c4bdcb1c4d49 2024-02-07T12:46:00.408 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (53s) 12s ago 22m 16.2M - 1.5.0 0da6a335fe13 04ea81f79d85 2024-02-07T12:46:00.408 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (26m) 12s ago 26m 54.4M 2993M 17.2.0 e1d6a67b021e 87f2560a9b15 2024-02-07T12:46:00.408 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (25m) 12s ago 25m 57.0M 2993M 17.2.0 e1d6a67b021e 0f9a190f0e0d 2024-02-07T12:46:00.408 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (25m) 12s ago 25m 52.7M 2993M 17.2.0 e1d6a67b021e 7db94f25a1aa 2024-02-07T12:46:00.408 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (25m) 12s ago 25m 53.2M 2993M 17.2.0 e1d6a67b021e 0b6b674188ed 2024-02-07T12:46:00.409 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (24m) 12s ago 24m 55.2M 3248M 17.2.0 e1d6a67b021e df72cfd69b3b 2024-02-07T12:46:00.409 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (24m) 12s ago 24m 52.4M 3248M 17.2.0 e1d6a67b021e 3bdb22c07537 2024-02-07T12:46:00.409 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (24m) 12s ago 24m 52.0M 3248M 17.2.0 e1d6a67b021e 7859b651e0d9 2024-02-07T12:46:00.409 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (23m) 12s ago 23m 55.1M 3248M 17.2.0 e1d6a67b021e b301e7c75d73 2024-02-07T12:46:00.409 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (35s) 12s ago 22m 47.1M - 2.43.0 a07b618ecd1d c9b607eea1be 2024-02-07T12:46:00.409 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (21m) 12s ago 21m 77.8M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:46:00.409 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (21m) 12s ago 21m 77.3M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:46:00.620 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 bash[66015]: aaffac63bc281628aa7d0fce29262aa3651c7c60651205efa0d467922a09b43c 2024-02-07T12:46:00.621 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 systemd[1]: Started Ceph grafana.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:46:00.621 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=settings t=2024-02-07T12:46:00.474407761Z level=info msg="Starting Grafana" version=9.4.12 commit=37ee25ba46 branch=HEAD compiled=2023-05-22T15:58:05Z 2024-02-07T12:46:00.621 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=settings t=2024-02-07T12:46:00.474643944Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-02-07T12:46:00.621 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=settings t=2024-02-07T12:46:00.474669437Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-02-07T12:46:00.621 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=settings t=2024-02-07T12:46:00.474682945Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-02-07T12:46:00.621 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=settings t=2024-02-07T12:46:00.474694195Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-02-07T12:46:00.621 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=settings t=2024-02-07T12:46:00.474706219Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-02-07T12:46:00.621 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=settings t=2024-02-07T12:46:00.474717334Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-02-07T12:46:00.621 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=settings t=2024-02-07T12:46:00.474729229Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-02-07T12:46:00.621 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=settings t=2024-02-07T12:46:00.474743743Z level=info msg="Path Home" path=/usr/share/grafana 2024-02-07T12:46:00.621 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=settings t=2024-02-07T12:46:00.474754022Z level=info msg="Path Data" path=/var/lib/grafana 2024-02-07T12:46:00.621 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=settings t=2024-02-07T12:46:00.474764066Z level=info msg="Path Logs" path=/var/log/grafana 2024-02-07T12:46:00.622 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=settings t=2024-02-07T12:46:00.474774147Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-02-07T12:46:00.622 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=settings t=2024-02-07T12:46:00.474784356Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-02-07T12:46:00.622 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=settings t=2024-02-07T12:46:00.474794681Z level=info msg="App mode production" 2024-02-07T12:46:00.622 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=sqlstore t=2024-02-07T12:46:00.474845541Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-02-07T12:46:00.622 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=sqlstore t=2024-02-07T12:46:00.474868845Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-02-07T12:46:00.622 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.486161796Z level=info msg="Starting DB migrations" 2024-02-07T12:46:00.622 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.489700046Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-02-07T12:46:00.623 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.498581207Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-02-07T12:46:00.623 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.499578952Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-02-07T12:46:00.623 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.502236302Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-02-07T12:46:00.623 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.505080646Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-02-07T12:46:00.623 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.506228555Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-02-07T12:46:00.623 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.508945643Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-02-07T12:46:00.623 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.511669551Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-02-07T12:46:00.623 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.514543391Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-02-07T12:46:00.624 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.515105473Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-02-07T12:46:00.624 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.517800868Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-02-07T12:46:00.624 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.520632114Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-02-07T12:46:00.624 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.523284463Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-02-07T12:46:00.624 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.526064219Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-02-07T12:46:00.624 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.528928055Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-02-07T12:46:00.624 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.531606837Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-02-07T12:46:00.624 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.534426083Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-02-07T12:46:00.624 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.535389705Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-02-07T12:46:00.624 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.536370073Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-02-07T12:46:00.624 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.536727395Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-02-07T12:46:00.624 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.537744707Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-02-07T12:46:00.624 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.53870496Z level=info msg="Executing migration" id="create provenance_type table" 2024-02-07T12:46:00.624 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.539142958Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-02-07T12:46:00.625 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.541782506Z level=info msg="Executing migration" id="create alert_image table" 2024-02-07T12:46:00.625 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.544600885Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-02-07T12:46:00.625 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.547318165Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-02-07T12:46:00.625 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.547697004Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-02-07T12:46:00.625 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.54813515Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-02-07T12:46:00.625 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.548572185Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-02-07T12:46:00.625 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.548676951Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-02-07T12:46:00.625 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.548973291Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-02-07T12:46:00.625 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.549505958Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-02-07T12:46:00.625 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.549983328Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-02-07T12:46:00.625 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.550274876Z level=info msg="Executing migration" id="create secrets table" 2024-02-07T12:46:00.625 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.550717024Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-02-07T12:46:00.625 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.555403947Z level=info msg="Executing migration" id="add name column into data_keys" 2024-02-07T12:46:00.625 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.556597244Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-02-07T12:46:00.626 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.557173342Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-02-07T12:46:00.626 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.562004831Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-02-07T12:46:00.626 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.566693259Z level=info msg="Executing migration" id="add column hidden to role table" 2024-02-07T12:46:00.626 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.567695111Z level=info msg="Executing migration" id="create query_history table v1" 2024-02-07T12:46:00.626 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.568533253Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-02-07T12:46:00.626 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.569333857Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-02-07T12:46:00.626 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.570145056Z level=info msg="Executing migration" id="teams permissions migration" 2024-02-07T12:46:00.626 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.571001406Z level=info msg="Executing migration" id="dashboard permissions" 2024-02-07T12:46:00.626 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.572387181Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-02-07T12:46:00.626 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.573311953Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-02-07T12:46:00.626 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.574140896Z level=info msg="Executing migration" id="alerting notification permissions" 2024-02-07T12:46:00.626 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.575004372Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-02-07T12:46:00.626 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.575774356Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-02-07T12:46:00.626 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.576619381Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-02-07T12:46:00.627 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.577646038Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-02-07T12:46:00.627 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.578537375Z level=info msg="Executing migration" id="create correlation table v1" 2024-02-07T12:46:00.627 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.579368273Z level=info msg="Executing migration" id="add index correlations.uid" 2024-02-07T12:46:00.627 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.580278885Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-02-07T12:46:00.627 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.581146996Z level=info msg="Executing migration" id="add correlation config column" 2024-02-07T12:46:00.627 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.582182869Z level=info msg="Executing migration" id="create entity_events table" 2024-02-07T12:46:00.627 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.583059602Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-02-07T12:46:00.627 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.583882298Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-02-07T12:46:00.627 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.583977298Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-02-07T12:46:00.627 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.584697519Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-02-07T12:46:00.627 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.584786211Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-02-07T12:46:00.627 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.585078132Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-02-07T12:46:00.627 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.585529127Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-02-07T12:46:00.627 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.59132606Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-02-07T12:46:00.627 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.591864106Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-02-07T12:46:00.628 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.592718551Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-02-07T12:46:00.628 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.593562126Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-02-07T12:46:00.628 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.59444731Z level=info msg="Executing migration" id="Drop public config table" 2024-02-07T12:46:00.628 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.595318537Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-02-07T12:46:00.628 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.596170607Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-02-07T12:46:00.628 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.59705693Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-02-07T12:46:00.628 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.597938579Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-02-07T12:46:00.628 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.598772351Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-02-07T12:46:00.628 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.601794394Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-02-07T12:46:00.628 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.604541598Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-02-07T12:46:00.628 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.60748906Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-02-07T12:46:00.628 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.610320717Z level=info msg="Executing migration" id="add share column" 2024-02-07T12:46:00.628 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.613050403Z level=info msg="Executing migration" id="create default alerting folders" 2024-02-07T12:46:00.628 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.615720089Z level=info msg="Executing migration" id="create file table" 2024-02-07T12:46:00.629 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.618512138Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-02-07T12:46:00.629 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.621303863Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-02-07T12:46:00.821 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:46:00.822 INFO:teuthology.orchestra.run.smithi019.stdout: "mon": { 2024-02-07T12:46:00.822 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-02-07T12:46:00.822 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:46:00.822 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr": { 2024-02-07T12:46:00.822 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:46:00.822 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:46:00.822 INFO:teuthology.orchestra.run.smithi019.stdout: "osd": { 2024-02-07T12:46:00.822 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-02-07T12:46:00.822 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:46:00.822 INFO:teuthology.orchestra.run.smithi019.stdout: "mds": {}, 2024-02-07T12:46:00.822 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw": { 2024-02-07T12:46:00.822 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:46:00.822 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:46:00.822 INFO:teuthology.orchestra.run.smithi019.stdout: "overall": { 2024-02-07T12:46:00.823 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-02-07T12:46:00.823 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:46:00.823 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:46:00.823 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:46:01.120 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.624105562Z level=info msg="Executing migration" id="create file_meta table" 2024-02-07T12:46:01.120 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.626997468Z level=info msg="Executing migration" id="file table idx: path key" 2024-02-07T12:46:01.120 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.62982442Z level=info msg="Executing migration" id="set path collation in file table" 2024-02-07T12:46:01.120 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.632671224Z level=info msg="Executing migration" id="managed permissions migration" 2024-02-07T12:46:01.120 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.635511617Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-02-07T12:46:01.120 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.638226617Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-02-07T12:46:01.120 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.641038048Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-02-07T12:46:01.120 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.643824658Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-02-07T12:46:01.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.646609351Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-02-07T12:46:01.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.649360041Z level=info msg="Executing migration" id="update group index for alert rules" 2024-02-07T12:46:01.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.652067108Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-02-07T12:46:01.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.654830676Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-02-07T12:46:01.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.657574807Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-02-07T12:46:01.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.660245437Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-02-07T12:46:01.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.66296223Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-02-07T12:46:01.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.665564488Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-02-07T12:46:01.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.675467516Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-02-07T12:46:01.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.678140563Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-02-07T12:46:01.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.68092144Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-02-07T12:46:01.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.68451367Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-02-07T12:46:01.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.687264248Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-02-07T12:46:01.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.690050894Z level=info msg="Executing migration" id="create folder table" 2024-02-07T12:46:01.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.692806166Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-02-07T12:46:01.122 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.695381402Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-02-07T12:46:01.122 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.698065252Z level=info msg="Executing migration" id="Update folder title length" 2024-02-07T12:46:01.122 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.70086542Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-02-07T12:46:01.122 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=migrator t=2024-02-07T12:46:00.703612248Z level=info msg="migrations completed" performed=105 skipped=376 duration=213.955131ms 2024-02-07T12:46:01.122 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=sqlstore t=2024-02-07T12:46:00.703932749Z level=info msg="Created default organization" 2024-02-07T12:46:01.122 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=plugin.loader t=2024-02-07T12:46:00.7606217Z level=info msg="Plugin registered" pluginID=input 2024-02-07T12:46:01.122 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=plugin.loader t=2024-02-07T12:46:00.788505253Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-02-07T12:46:01.122 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=plugin.loader t=2024-02-07T12:46:00.788536606Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-02-07T12:46:01.122 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=secrets t=2024-02-07T12:46:00.788575444Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-02-07T12:46:01.122 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=query_data t=2024-02-07T12:46:00.790728416Z level=info msg="Query Service initialization" 2024-02-07T12:46:01.122 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=live.push_http t=2024-02-07T12:46:00.795690433Z level=info msg="Live Push Gateway initialization" 2024-02-07T12:46:01.158 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:46:01.158 INFO:teuthology.orchestra.run.smithi019.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:46:01.158 INFO:teuthology.orchestra.run.smithi019.stdout: "in_progress": true, 2024-02-07T12:46:01.158 INFO:teuthology.orchestra.run.smithi019.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-02-07T12:46:01.158 INFO:teuthology.orchestra.run.smithi019.stdout: "services_complete": [ 2024-02-07T12:46:01.159 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr" 2024-02-07T12:46:01.159 INFO:teuthology.orchestra.run.smithi019.stdout: ], 2024-02-07T12:46:01.159 INFO:teuthology.orchestra.run.smithi019.stdout: "progress": "2/2 daemons upgraded", 2024-02-07T12:46:01.159 INFO:teuthology.orchestra.run.smithi019.stdout: "message": "Currently upgrading grafana daemons", 2024-02-07T12:46:01.159 INFO:teuthology.orchestra.run.smithi019.stdout: "is_paused": false 2024-02-07T12:46:01.159 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:46:01.506 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:01 smithi172 ceph-mon[25457]: from='client.25351 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:01.506 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:01 smithi172 ceph-mon[25457]: from='client.15471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:01.506 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:01 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:46:01.506 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:01 smithi172 ceph-mon[25457]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:46:01.506 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:01 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:01.507 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:01 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:01.507 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:01 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:46:01.507 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:01 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/538136000' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:01.507 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=infra.usagestats.collector t=2024-02-07T12:46:01.480248746Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-02-07T12:46:01.507 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=provisioning.datasources t=2024-02-07T12:46:01.482291674Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-02-07T12:46:01.507 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=provisioning.datasources t=2024-02-07T12:46:01.482325721Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-02-07T12:46:01.507 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=provisioning.datasources t=2024-02-07T12:46:01.482431716Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-02-07T12:46:01.507 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=provisioning.alerting t=2024-02-07T12:46:01.504593723Z level=info msg="starting to provision alerting" 2024-02-07T12:46:01.507 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=provisioning.alerting t=2024-02-07T12:46:01.504614075Z level=info msg="finished to provision alerting" 2024-02-07T12:46:01.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:01 smithi019 ceph-mon[29267]: from='client.25351 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:01.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:01 smithi019 ceph-mon[29267]: from='client.15471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:01.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:01 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:46:01.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:01 smithi019 ceph-mon[29267]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:46:01.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:01 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:01.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:01 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:01.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:01 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:46:01.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:01 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/538136000' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:01.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:01 smithi019 ceph-mon[24574]: from='client.25351 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:01.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:01 smithi019 ceph-mon[24574]: from='client.15471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:01.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:01 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:46:01.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:01 smithi019 ceph-mon[24574]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:46:01.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:01 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:01.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:01 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:01.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:01 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:46:01.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:01 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/538136000' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:01.773 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=grafanaStorageLogger t=2024-02-07T12:46:01.504995268Z level=info msg="storage starting" 2024-02-07T12:46:01.773 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=ngalert.state.manager t=2024-02-07T12:46:01.505297739Z level=info msg="Warming state cache for startup" 2024-02-07T12:46:01.774 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=http.server t=2024-02-07T12:46:01.507337206Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-02-07T12:46:01.774 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=ngalert.state.manager t=2024-02-07T12:46:01.519023483Z level=info msg="State cache has been initialized" states=0 duration=13.724861ms 2024-02-07T12:46:01.774 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=ticker t=2024-02-07T12:46:01.519087852Z level=info msg=starting first_tick=2024-02-07T12:46:10Z 2024-02-07T12:46:01.774 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=ngalert.multiorg.alertmanager t=2024-02-07T12:46:01.519083432Z level=info msg="starting MultiOrg Alertmanager" 2024-02-07T12:46:01.774 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=sqlstore.transactions t=2024-02-07T12:46:01.537312297Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-02-07T12:46:01.774 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:46:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=sqlstore.transactions t=2024-02-07T12:46:01.549481569Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-02-07T12:46:02.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:02 smithi172 ceph-mon[25457]: from='client.25363 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:02.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:02 smithi172 ceph-mon[25457]: from='client.15489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:02.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:02 smithi019 ceph-mon[29267]: from='client.25363 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:02.673 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:02 smithi019 ceph-mon[29267]: from='client.15489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:02.673 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:02 smithi019 ceph-mon[24574]: from='client.25363 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:02.674 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:02 smithi019 ceph-mon[24574]: from='client.15489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:03.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:03 smithi172 ceph-mon[25457]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:46:03.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:03 smithi019 ceph-mon[29267]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:46:03.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:03 smithi019 ceph-mon[24574]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:46:04.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:46:03] "GET /metrics HTTP/1.1" 200 37984 "" "Prometheus/2.43.0" 2024-02-07T12:46:05.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:05 smithi019 ceph-mon[24574]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:05.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:05 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:05.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:05 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:05.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:05 smithi019 ceph-mon[29267]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:05.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:05 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:05.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:05 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:05.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:05 smithi172 ceph-mon[25457]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:05.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:05 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:05.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:05 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:07.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:06 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:07.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:06 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:07.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:06 smithi172 ceph-mon[25457]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:46:07.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:06 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:07.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:06 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:07.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:06 smithi019 ceph-mon[24574]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:46:07.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:06 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:07.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:06 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:07.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:06 smithi019 ceph-mon[29267]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:46:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:09 smithi172 ceph-mon[25457]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:09 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:46:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:09 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:09.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:09 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:09.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:09 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:46:09.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:09 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:46:09.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:09 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:09.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:09 smithi019 ceph-mon[29267]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:09.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:09 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:46:09.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:09 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:09.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:09 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:09.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:09 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:46:09.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:09 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:46:09.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:09 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:09.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:09 smithi019 ceph-mon[24574]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:09.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:09 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:46:09.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:09 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:09.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:09 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:09.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:09 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:46:09.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:09 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:46:09.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:09 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:10.552 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:10 smithi019 ceph-mon[29267]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 929 B/s rd, 0 op/s 2024-02-07T12:46:10.552 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:10 smithi019 ceph-mon[29267]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 567 B/s rd, 0 op/s 2024-02-07T12:46:10.552 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:10 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:46:10.552 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:10 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:46:10.553 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:10 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:46:10.553 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:10 smithi019 ceph-mon[24574]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 929 B/s rd, 0 op/s 2024-02-07T12:46:10.553 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:10 smithi019 ceph-mon[24574]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 567 B/s rd, 0 op/s 2024-02-07T12:46:10.553 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:10 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:46:10.553 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:10 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:46:10.553 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:10 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:46:10.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:10 smithi172 ceph-mon[25457]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 929 B/s rd, 0 op/s 2024-02-07T12:46:10.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:10 smithi172 ceph-mon[25457]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 567 B/s rd, 0 op/s 2024-02-07T12:46:10.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:10 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:46:10.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:10 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:46:10.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:10 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:46:11.234 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:46:11.099+0000 7eff60d5c700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:11.234 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:46:11.235 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:11.235 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:11.235 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:11.235 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:11.235 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:11.235 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:46:11.235 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:11.235 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:46:11.235 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:11.235 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:46:11.235 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:11.235 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:11.235 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:11.236 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:11.236 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:11.236 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:11.236 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:11.236 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:11.236 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:11.237 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:46:11.237 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:46:11.237 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:46:11.237 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:46:11.237 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:46:11.237 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:46:11.237 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:46:11.237 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:46:11.237 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:46:11.237 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:46:11.237 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:46:11.237 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:46:11.237 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:46:11.237 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:46:11.237 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:46:11.238 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:46:11.238 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:46:11.238 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:11.238 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:46:11.238 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:11.238 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:11.238 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:11.238 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:11.238 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:11.238 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:46:11.238 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:11.238 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:46:11.238 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:11.238 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:46:11.239 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:11.239 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:11.239 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:11.239 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:11.239 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:11.239 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:11.239 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:11.239 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:11.239 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:11.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:46:11.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:46:11.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:46:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:46:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:46:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:11.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:46:11.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:46:11.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-02-07T12:46:11.599 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.599 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.599 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-02-07T12:46:11.599 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:46:11.600 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:46:11.600 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:46:11.600 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:46:11.600 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:46:11.600 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:46:11.600 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.600 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.600 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.600 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.600 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.601 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.601 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.601 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.601 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.601 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.601 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.601 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.601 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.601 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.601 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:11.601 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:46:11.601 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:46:11.601 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-02-07T12:46:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-02-07T12:46:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:46:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:46:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-02-07T12:46:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:46:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:46:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:46:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:46:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-02-07T12:46:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:46:11.602 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:46:11.603 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:46:11.603 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:46:11.603 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-02-07T12:46:11.603 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.603 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.603 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:46:11.603 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:46:11.603 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-02-07T12:46:11.603 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:46:11.604 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:46:11.604 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:46:11.604 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:46:11.604 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-02-07T12:46:11.604 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:46:11.604 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:46:11.604 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:46:11.604 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-02-07T12:46:11.604 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:46:11.604 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:46:11.605 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:46:11.605 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:46:11.605 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-02-07T12:46:11.605 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:46:11.605 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:46:11.605 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:46:11.605 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:46:11.605 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-02-07T12:46:11.605 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.605 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.605 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:46:11.605 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:46:11.605 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-02-07T12:46:11.606 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:46:11.606 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:46:11.606 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:46:11.606 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:46:11.606 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-02-07T12:46:11.606 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:46:11.606 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:46:11.606 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-02-07T12:46:11.607 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.607 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.607 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.607 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.607 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.607 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.607 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.607 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.607 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.607 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.607 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.607 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.608 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:46:11.608 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:46:11.608 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-02-07T12:46:11.608 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:46:11.608 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:46:11.608 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:46:11.608 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:11.608 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-02-07T12:46:11.608 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.609 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.609 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.609 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.609 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.609 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.609 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.609 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.609 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.609 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.609 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.609 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.609 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:46:11.609 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:46:11.609 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-02-07T12:46:11.609 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:46:11.610 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:46:11.610 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:46:11.610 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:46:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:46:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:46:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:46:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:46:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:46:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:46:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-02-07T12:46:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-02-07T12:46:11.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:46:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:46:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-02-07T12:46:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:46:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:46:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:46:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:46:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-02-07T12:46:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:46:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:46:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:46:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:46:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-02-07T12:46:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:46:11.622 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:46:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-02-07T12:46:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:46:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:46:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:46:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:46:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-02-07T12:46:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:46:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:46:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-02-07T12:46:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:46:11.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:46:11.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:46:11.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-02-07T12:46:11.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:46:11.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:46:11.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:46:11.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:11 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:12.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:12 smithi019 ceph-mon[29267]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:12.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:46:12.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:12.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:12.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:12.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:12.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:12.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:46:12.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:12.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:46:12.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:12.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:46:12.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:12.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:12.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:12.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:12.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:12.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:12.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:12.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:12.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:12.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:46:12.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:46:12.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:46:12.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:46:12.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:46:12.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:46:12.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:46:12.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:46:12.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:46:12.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:46:12.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:46:12.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:46:12.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:46:12.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:46:12.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:46:12.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:46:12.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:46:12.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:12.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:46:12.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:12.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:12.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:12.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:12.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:12.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:46:12.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:12.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:46:12.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:12.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:46:12.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:12.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:12.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:12.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:12.598 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:12 smithi019 ceph-mon[24574]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:12.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:46:12.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:12.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:12.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:12.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:12.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:12.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:46:12.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:12.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:46:12.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:12.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:46:12.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:12.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:12.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:12.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:12.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:12.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:12.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:12.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:12.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:12.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:46:12.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:46:12.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:46:12.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:46:12.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:46:12.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:46:12.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:46:12.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:46:12.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:46:12.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:46:12.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:46:12.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:46:12.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:46:12.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:46:12.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:46:12.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:46:12.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:46:12.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:12.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:46:12.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:12.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:12.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:12.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:12.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:12.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:46:12.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:12.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:46:12.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:12.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:46:12.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:12.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:12.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:12.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:12.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:12.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:12.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:12.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:12.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:12.603 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:12 smithi019 ceph-mon[29267]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:46:12.603 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:12 smithi019 ceph-mon[29267]: Upgrade: Finalizing container_image settings 2024-02-07T12:46:12.603 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:12 smithi019 ceph-mon[29267]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:46:12.603 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:12 smithi019 ceph-mon[29267]: Upgrade: Complete! 2024-02-07T12:46:12.603 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:12 smithi019 ceph-mon[29267]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-02-07T12:46:12.603 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:12 smithi019 ceph-mon[29267]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1007 B/s rd, 0 op/s 2024-02-07T12:46:12.603 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:12 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:46:12.603 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:12 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:46:12.603 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:12 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:46:12.604 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:12.604 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:12.604 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:12.604 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:12.604 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:12.604 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:12 smithi019 ceph-mon[24574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:46:12.604 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:12 smithi019 ceph-mon[24574]: Upgrade: Finalizing container_image settings 2024-02-07T12:46:12.604 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:12 smithi019 ceph-mon[24574]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:46:12.604 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:12 smithi019 ceph-mon[24574]: Upgrade: Complete! 2024-02-07T12:46:12.604 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:12 smithi019 ceph-mon[24574]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-02-07T12:46:12.604 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:12 smithi019 ceph-mon[24574]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1007 B/s rd, 0 op/s 2024-02-07T12:46:12.605 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:12 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:46:12.605 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:12 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:46:12.605 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:12 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:46:12.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:12 smithi172 ceph-mon[25457]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:12.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:46:12.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:12.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:12.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:12.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:12.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:12.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:46:12.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:12.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:46:12.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:12.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:46:12.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:12.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:12.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:12.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:12.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:12.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:12.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:12.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:12.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:12.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:46:12.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:46:12.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:46:12.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:46:12.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:46:12.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:46:12.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:46:12.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:46:12.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:46:12.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:46:12.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:46:12.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:46:12.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:46:12.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:46:12.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:46:12.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:46:12.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:46:12.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:12.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:46:12.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:12.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:12.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:12.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:12.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:12.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:46:12.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:12.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:46:12.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:12.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:46:12.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:12.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:12.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:12.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:12.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:12.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:12.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:12.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:12.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:12.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:12 smithi172 ceph-mon[25457]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:46:12.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:12 smithi172 ceph-mon[25457]: Upgrade: Finalizing container_image settings 2024-02-07T12:46:12.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:12 smithi172 ceph-mon[25457]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:46:12.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:12 smithi172 ceph-mon[25457]: Upgrade: Complete! 2024-02-07T12:46:12.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:12 smithi172 ceph-mon[25457]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-02-07T12:46:12.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:12 smithi172 ceph-mon[25457]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1007 B/s rd, 0 op/s 2024-02-07T12:46:12.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:12 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:46:12.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:12 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:46:12.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:12 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:46:13.015 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:46:13.010+0000 7eff60d5c700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:13.015 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:46:13.015 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:13.015 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:13.015 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:13.015 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:13.015 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:13.015 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:46:13.015 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:13.015 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:46:13.016 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:13.016 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:46:13.016 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:13.016 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:13.016 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:13.016 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:13.016 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:13.016 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:13.016 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:13.017 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:13.017 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:13.017 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:46:13.017 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:46:13.017 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:46:13.017 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:46:13.017 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:46:13.017 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:46:13.017 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:46:13.018 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:46:13.018 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:46:13.018 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:46:13.018 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:46:13.018 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:46:13.018 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:46:13.018 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:46:13.018 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:46:13.018 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:46:13.018 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:46:13.018 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:13.018 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:46:13.018 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:13.018 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:13.019 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:13.019 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:13.019 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:13.019 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:46:13.019 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:13.019 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:46:13.019 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:13.019 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:46:13.019 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:13.019 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:13.019 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:13.019 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:13.019 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:13.019 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:13.019 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:13.020 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:13.020 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:13.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-mon[29267]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:46:13.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-mon[29267]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:46:13.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:46:13.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:46:13.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:46:13.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:13.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:46:13.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:46:13.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:46:13.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-mon[24574]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:46:13.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-mon[24574]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:46:13.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:46:13.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:46:13.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:46:13.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:13.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:46:13.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:46:13.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:46:13.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:13 smithi172 ceph-mon[25457]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:46:13.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:13 smithi172 ceph-mon[25457]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:46:13.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:13 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:46:13.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:13 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:46:13.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:13 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:46:13.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:13 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:13.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:13 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:46:13.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:13 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:46:13.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:13 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:46:14.039 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:46:13] "GET /metrics HTTP/1.1" 200 37982 "" "Prometheus/2.43.0" 2024-02-07T12:46:14.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-mon[29267]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:46:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:46:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:46:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:46:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:14.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:14.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:14.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:14.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:14.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:14.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:14.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:14.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:14.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:46:14.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:46:14.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:46:14.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:46:14.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:46:14.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:46:14.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:46:14.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:46:14.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:46:14.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:46:14.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:46:14.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:46:14.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:46:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:46:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:46:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:46:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:46:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:46:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:46:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:14.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:46:14.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:14.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:46:14.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:14.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:14.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:14.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:14.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-mon[24574]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:14.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:46:14.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:14.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:46:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:46:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:46:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:14.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:46:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:46:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:46:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:46:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:46:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:46:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:46:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:46:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:46:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:46:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:46:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:46:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:46:14.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:46:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:46:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:46:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:46:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:46:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:46:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:46:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:46:14.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:14.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:14.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:14.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:14.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:14.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:14.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:14.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:14.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:14.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-mon[29267]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:46:14.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-mon[29267]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-02-07T12:46:14.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-mon[29267]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:46:14.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-mon[29267]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:46:14.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:14.354 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:14.354 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:14.354 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:14.354 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:14.354 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:14.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-mon[24574]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:46:14.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-mon[24574]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-02-07T12:46:14.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-mon[24574]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:46:14.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-mon[24574]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:46:14.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:14.521 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:14 smithi172 ceph-mon[25457]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:14.521 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:46:14.521 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:14.522 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:14.522 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:14.522 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:14.522 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:14.522 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:46:14.522 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:14.522 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:46:14.522 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:14.522 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:46:14.522 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:14.522 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:14.522 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:14.522 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:14.522 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:14.522 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:14.522 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:14.523 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:14.523 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:14.523 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:46:14.523 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:46:14.523 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:46:14.523 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:46:14.523 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:46:14.523 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:46:14.523 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:46:14.523 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:46:14.523 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:46:14.523 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:46:14.523 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:46:14.523 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:46:14.523 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:46:14.523 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:46:14.523 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:46:14.524 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:46:14.524 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:46:14.524 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:14.524 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:46:14.524 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:14.524 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:14.524 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:14.524 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:14.524 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:14.524 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:46:14.524 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:14.524 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:46:14.524 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:14.525 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:46:14.525 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:14.525 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:14.525 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:14.525 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:14.525 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:14.525 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:14.525 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:14.525 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:14.525 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:14.525 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:14 smithi172 ceph-mon[25457]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:46:14.525 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:14 smithi172 ceph-mon[25457]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-02-07T12:46:14.525 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:14 smithi172 ceph-mon[25457]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:46:14.525 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:14 smithi172 ceph-mon[25457]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:46:14.525 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:14 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:15.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:46:14.843+0000 7eff60d5c700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:15.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:46:15.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:15.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:15.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:15.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:15.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:15.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:46:15.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:15.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:46:15.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:15.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:46:15.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:15.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:15.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:15.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:15.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:15.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:15.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:15.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:15.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:15.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:46:15.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:46:15.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:46:15.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:46:15.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:46:15.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:46:15.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:46:15.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:46:15.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:46:15.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:46:15.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:46:15.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:46:15.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:46:15.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:46:15.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:46:15.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:46:15.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:46:15.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:15.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:46:15.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:15.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:15.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:15.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:15.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:15.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:46:15.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:15.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:46:15.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:15.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:46:15.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:15.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:15.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:15.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:15.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:15.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:15.100 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:15.100 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:15.100 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:14 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:16.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:16 smithi019 ceph-mon[24574]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:16.594 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:46:16.594 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:16.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:16.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:16.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:16.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:16.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:46:16.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:16.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:46:16.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:16.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:46:16.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:16.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:16.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:16.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:16.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:16.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:16.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:16.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:16.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:16.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:46:16.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:46:16.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:46:16.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:46:16.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:46:16.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:46:16.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:46:16.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:46:16.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:46:16.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:46:16.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:46:16.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:46:16.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:46:16.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:46:16.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:46:16.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:46:16.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:46:16.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:16.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:46:16.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:16.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:16.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:16.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:16.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:16.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:46:16.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:16.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:46:16.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:16.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:46:16.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:16.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:16.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:16.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:16.598 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:16 smithi019 ceph-mon[29267]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:16.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:46:16.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:16.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:16.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:16.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:16.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:16.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:46:16.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:16.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:46:16.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:16.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:46:16.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:16.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:16.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:16.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:16.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:16.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:16.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:16.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:16.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:16.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:46:16.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:46:16.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:46:16.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:46:16.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:46:16.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:46:16.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:46:16.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:46:16.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:46:16.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:46:16.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:46:16.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:46:16.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:46:16.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:46:16.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:46:16.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:46:16.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:46:16.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:16.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:46:16.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:16.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:16.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:16.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:16.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:16.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:46:16.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:16.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:46:16.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:16.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:46:16.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:16.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:16.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:16.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:16.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:16.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:16.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:16.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:16.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:16.602 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:16 smithi019 ceph-mon[29267]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-02-07T12:46:16.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:16.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:16.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:16.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:16.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:16.602 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:16 smithi019 ceph-mon[24574]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-02-07T12:46:16.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:16 smithi172 ceph-mon[25457]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:16.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:46:16.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:16.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:16.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:16.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:16.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:16.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:46:16.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:16.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:46:16.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:16.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:46:16.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:16.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:16.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:16.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:16.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:16.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:16.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:16.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:16.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:16.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:46:16.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:46:16.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:46:16.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:46:16.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:46:16.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:46:16.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:46:16.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:46:16.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:46:16.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:46:16.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:46:16.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:46:16.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:46:16.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:46:16.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:46:16.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:46:16.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:46:16.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:16.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:46:16.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:16.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:16.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:16.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:16.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:16.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:46:16.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:16.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:46:16.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:16.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:46:16.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:16.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:16.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:16.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:16.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:16.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:16.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:16.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:16.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:16.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:16 smithi172 ceph-mon[25457]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-02-07T12:46:18.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:18 smithi019 ceph-mon[24574]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 870 B/s rd, 0 op/s 2024-02-07T12:46:18.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:18 smithi019 ceph-mon[29267]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 870 B/s rd, 0 op/s 2024-02-07T12:46:18.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:18 smithi172 ceph-mon[25457]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 870 B/s rd, 0 op/s 2024-02-07T12:46:20.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:20 smithi019 ceph-mon[24574]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 655 B/s rd, 0 op/s 2024-02-07T12:46:20.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:20 smithi019 ceph-mon[29267]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 655 B/s rd, 0 op/s 2024-02-07T12:46:20.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:20 smithi172 ceph-mon[25457]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 655 B/s rd, 0 op/s 2024-02-07T12:46:21.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:21 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:46:21.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:21 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:46:21.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:21 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:46:22.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:22 smithi019 ceph-mon[29267]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:46:22.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:22 smithi019 ceph-mon[24574]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:46:22.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:22 smithi172 ceph-mon[25457]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:46:24.261 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:46:23] "GET /metrics HTTP/1.1" 200 37982 "" "Prometheus/2.43.0" 2024-02-07T12:46:24.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:24 smithi019 ceph-mon[29267]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:46:24.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:24 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:46:24.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:24 smithi019 ceph-mon[24574]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:46:24.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:24 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:46:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:24 smithi172 ceph-mon[25457]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:46:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:24 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:46:26.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:26 smithi019 ceph-mon[29267]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:46:26.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:26 smithi019 ceph-mon[24574]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:46:26.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:26 smithi172 ceph-mon[25457]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:46:28.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:28 smithi019 ceph-mon[29267]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:28.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:28 smithi019 ceph-mon[24574]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:28.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:28 smithi172 ceph-mon[25457]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:30.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:30 smithi019 ceph-mon[24574]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:30.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:30 smithi019 ceph-mon[29267]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:30.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:30 smithi172 ceph-mon[25457]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:31.525 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:31 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:46:31.525 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:31 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:46:31.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:31 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:46:32.173 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-02-07T12:46:32.487 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:32 smithi019 ceph-mon[29267]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:46:32.487 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:32 smithi019 ceph-mon[24574]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:46:32.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:32 smithi172 ceph-mon[25457]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:46:33.557 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:33 smithi019 ceph-mon[29267]: from='client.25381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:33.557 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:33 smithi019 ceph-mon[24574]: from='client.25381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:33.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:33 smithi172 ceph-mon[25457]: from='client.25381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:34.201 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:46:34.201 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:46:33] "GET /metrics HTTP/1.1" 200 37982 "" "Prometheus/2.43.0" 2024-02-07T12:46:34.535 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:34 smithi019 ceph-mon[24574]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:34.535 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:34 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/686135457' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:34.536 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:34 smithi019 ceph-mon[29267]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:34.536 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:34 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/686135457' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:34.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:34 smithi172 ceph-mon[25457]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:34.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:34 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/686135457' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:34.860 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph versions | jq -e '"'"'.mgr | keys'"'"' | grep $sha1' 2024-02-07T12:46:35.407 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:35 smithi019 ceph-mon[29267]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:46:35.407 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:35 smithi019 ceph-mon[24574]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:46:35.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:35 smithi172 ceph-mon[25457]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:46:36.958 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)" 2024-02-07T12:46:37.250 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:36 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2881632406' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:37.250 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:36 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2881632406' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:37.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:36 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2881632406' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:37.544 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 2'"'"'' 2024-02-07T12:46:38.187 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:38 smithi019 ceph-mon[29267]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:38.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:38 smithi019 ceph-mon[24574]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:38.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:38 smithi172 ceph-mon[25457]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:39.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:39 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:46:39.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:39 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:46:39.587 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:46:39.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:39 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:46:40.210 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 2'"'"'' 2024-02-07T12:46:40.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:40 smithi019 ceph-mon[29267]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:40.527 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:40 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/3734349099' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:40.527 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:40 smithi019 ceph-mon[24574]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:40.527 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:40 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/3734349099' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:40.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:40 smithi172 ceph-mon[25457]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:40.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:40 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/3734349099' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:41.507 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:41 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:46:41.508 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:41 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:46:41.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:41 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:46:42.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:42 smithi019 ceph-mon[29267]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:46:42.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:42 smithi019 ceph-mon[24574]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:46:42.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:42 smithi172 ceph-mon[25457]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:46:43.309 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:43 smithi019 ceph-mon[24574]: from='client.25405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:43.309 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:43 smithi019 ceph-mon[29267]: from='client.25405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:43.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:43 smithi172 ceph-mon[25457]: from='client.25405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:44.156 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:46:44.192 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:46:43] "GET /metrics HTTP/1.1" 200 37993 "" "Prometheus/2.43.0" 2024-02-07T12:46:44.520 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:44 smithi172 ceph-mon[25457]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:44.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:44 smithi019 ceph-mon[29267]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:44.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:44 smithi019 ceph-mon[24574]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:44.742 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch upgrade status' 2024-02-07T12:46:46.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:46 smithi019 ceph-mon[24574]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:46:46.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:46 smithi019 ceph-mon[29267]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:46:46.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:46 smithi172 ceph-mon[25457]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:46:46.690 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:46:46.690 INFO:teuthology.orchestra.run.smithi019.stdout: "target_image": null, 2024-02-07T12:46:46.690 INFO:teuthology.orchestra.run.smithi019.stdout: "in_progress": false, 2024-02-07T12:46:46.690 INFO:teuthology.orchestra.run.smithi019.stdout: "which": "", 2024-02-07T12:46:46.691 INFO:teuthology.orchestra.run.smithi019.stdout: "services_complete": [], 2024-02-07T12:46:46.691 INFO:teuthology.orchestra.run.smithi019.stdout: "progress": null, 2024-02-07T12:46:46.691 INFO:teuthology.orchestra.run.smithi019.stdout: "message": "", 2024-02-07T12:46:46.691 INFO:teuthology.orchestra.run.smithi019.stdout: "is_paused": false 2024-02-07T12:46:46.691 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:46:47.292 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph health detail' 2024-02-07T12:46:48.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:48 smithi019 ceph-mon[29267]: from='client.25411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:48.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:48 smithi019 ceph-mon[29267]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:48.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:48 smithi019 ceph-mon[24574]: from='client.25411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:48.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:48 smithi019 ceph-mon[24574]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:48.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:48 smithi172 ceph-mon[25457]: from='client.25411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:48.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:48 smithi172 ceph-mon[25457]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:49.232 INFO:teuthology.orchestra.run.smithi019.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-02-07T12:46:49.232 INFO:teuthology.orchestra.run.smithi019.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-02-07T12:46:49.232 INFO:teuthology.orchestra.run.smithi019.stdout: daemon iscsi.foo.smithi019.svjimj on smithi019 is in unknown state 2024-02-07T12:46:49.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:49 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/104964479' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:46:49.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:49 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/104964479' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:46:49.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:49 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/104964479' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:46:49.861 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '"'"'{print $2}'"'"')' 2024-02-07T12:46:50.408 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:50 smithi019 ceph-mon[24574]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:50.409 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:50 smithi019 ceph-mon[29267]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:50.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:50 smithi172 ceph-mon[25457]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:51.278 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:51 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:46:51.278 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:51 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:46:51.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:51 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:46:52.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:52 smithi019 ceph-mon[29267]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:46:52.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:52 smithi019 ceph-mon[24574]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:46:52.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:52 smithi172 ceph-mon[25457]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:46:53.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:53 smithi019 ceph-mon[29267]: from='client.15537 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:53.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:53 smithi019 ceph-mon[29267]: from='client.15543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "daemon_types": "mon", "hosts": "smithi172", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:53.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:53 smithi019 ceph-mon[24574]: from='client.15537 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:53.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:53 smithi019 ceph-mon[24574]: from='client.15543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "daemon_types": "mon", "hosts": "smithi172", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:53.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:53 smithi172 ceph-mon[25457]: from='client.15537 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:53.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:53 smithi172 ceph-mon[25457]: from='client.15543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "daemon_types": "mon", "hosts": "smithi172", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:54.153 INFO:teuthology.orchestra.run.smithi019.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:46:54.169 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:46:53] "GET /metrics HTTP/1.1" 200 37993 "" "Prometheus/2.43.0" 2024-02-07T12:46:54.454 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:54 smithi019 ceph-mon[24574]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:54.454 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:54 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:46:54.454 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:54 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:54.454 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:54 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:46:54.454 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:54 smithi019 ceph-mon[29267]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:54.454 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:54 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:46:54.455 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:54 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:54.455 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:54 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:46:54.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:54 smithi172 ceph-mon[25457]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:46:54.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:54 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:46:54.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:54 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:54.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:54 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:46:54.787 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-02-07T12:46:55.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:55 smithi019 ceph-mon[29267]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:46:55.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:55 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:46:55.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:55 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:46:55.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:55 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:55.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:55 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:46:55.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:55 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:46:55.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:55 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:46:55.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:55 smithi019 ceph-mon[24574]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:46:55.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:55 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:46:55.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:55 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:46:55.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:55 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:55.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:55 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:46:55.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:55 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:46:55.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:55 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:46:55.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:55 smithi172 ceph-mon[25457]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:46:55.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:55 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:46:55.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:55 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:46:55.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:55 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:46:55.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:55 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:46:55.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:55 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:46:55.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:55 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:46:56.406 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:56 smithi019 ceph-mon[29267]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 860 B/s rd, 0 op/s 2024-02-07T12:46:56.406 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:56 smithi019 ceph-mon[29267]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 517 B/s rd, 0 op/s 2024-02-07T12:46:56.406 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:56 smithi019 ceph-mon[29267]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:46:56.406 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:56 smithi019 ceph-mon[29267]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:46:56.406 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:56 smithi019 ceph-mon[24574]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 860 B/s rd, 0 op/s 2024-02-07T12:46:56.406 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:56 smithi019 ceph-mon[24574]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 517 B/s rd, 0 op/s 2024-02-07T12:46:56.406 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:56 smithi019 ceph-mon[24574]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:46:56.407 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:56 smithi019 ceph-mon[24574]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:46:56.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:56 smithi172 ceph-mon[25457]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 860 B/s rd, 0 op/s 2024-02-07T12:46:56.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:56 smithi172 ceph-mon[25457]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 517 B/s rd, 0 op/s 2024-02-07T12:46:56.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:56 smithi172 ceph-mon[25457]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:46:56.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:56 smithi172 ceph-mon[25457]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:46:56.988 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:46:57.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-mon[24574]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:46:57.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-mon[24574]: from='client.25435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:57.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-mon[29267]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:46:57.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-mon[29267]: from='client.25435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:57.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:57 smithi172 ceph-mon[25457]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:46:57.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:57 smithi172 ceph-mon[25457]: from='client.25435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:57.664 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:46:57.664 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (75s) 70s ago 23m 17.3M - 0.25.0 c8568f914cd2 fec9255f35e7 2024-02-07T12:46:57.664 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (57s) 52s ago 22m 50.6M - 9.4.12 a0ea5e3c46d9 aaffac63bc28 2024-02-07T12:46:57.665 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 unknown 70s ago 22m 57.2M - 3.5 e1d6a67b021e 5113aa77e3f2 2024-02-07T12:46:57.665 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283,8765 running (2m) 52s ago 27m 451M - 19.0.0-1252-g44458cce 79957b372307 5aa0c04fdfd7 2024-02-07T12:46:57.665 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (12m) 70s ago 29m 503M - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:46:57.665 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (29m) 70s ago 29m 79.6M 2048M 17.2.0 e1d6a67b021e f3a8a050cf1d 2024-02-07T12:46:57.665 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (27m) 52s ago 27m 57.6M 2048M 17.2.0 e1d6a67b021e 7d62c7c4d075 2024-02-07T12:46:57.665 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (27m) 70s ago 27m 61.4M 2048M 17.2.0 e1d6a67b021e b9825d0db32d 2024-02-07T12:46:57.665 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (117s) 70s ago 23m 8632k - 1.5.0 0da6a335fe13 c4bdcb1c4d49 2024-02-07T12:46:57.665 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (110s) 52s ago 23m 16.3M - 1.5.0 0da6a335fe13 04ea81f79d85 2024-02-07T12:46:57.665 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (27m) 70s ago 27m 54.4M 2993M 17.2.0 e1d6a67b021e 87f2560a9b15 2024-02-07T12:46:57.665 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (26m) 70s ago 26m 57.0M 2993M 17.2.0 e1d6a67b021e 0f9a190f0e0d 2024-02-07T12:46:57.665 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (26m) 70s ago 26m 52.7M 2993M 17.2.0 e1d6a67b021e 7db94f25a1aa 2024-02-07T12:46:57.665 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (26m) 70s ago 26m 53.2M 2993M 17.2.0 e1d6a67b021e 0b6b674188ed 2024-02-07T12:46:57.665 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (25m) 52s ago 25m 55.2M 3248M 17.2.0 e1d6a67b021e df72cfd69b3b 2024-02-07T12:46:57.665 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (25m) 52s ago 25m 52.4M 3248M 17.2.0 e1d6a67b021e 3bdb22c07537 2024-02-07T12:46:57.665 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (24m) 52s ago 24m 52.0M 3248M 17.2.0 e1d6a67b021e 7859b651e0d9 2024-02-07T12:46:57.666 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (24m) 52s ago 24m 55.1M 3248M 17.2.0 e1d6a67b021e b301e7c75d73 2024-02-07T12:46:57.666 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (92s) 52s ago 23m 47.3M - 2.43.0 a07b618ecd1d c9b607eea1be 2024-02-07T12:46:57.666 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (22m) 70s ago 22m 77.8M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:46:57.666 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (22m) 52s ago 22m 77.4M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:46:58.090 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:46:57.898+0000 7eff60d5c700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:58.090 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:46:58.090 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:58.090 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:58.091 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:58.091 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:58.091 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:58.091 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:46:58.091 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:58.091 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:46:58.091 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:58.091 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:46:58.091 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:58.091 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:58.091 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:58.091 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:58.091 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:58.091 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:58.092 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:58.092 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:46:58.092 INFO:teuthology.orchestra.run.smithi019.stdout: "mon": { 2024-02-07T12:46:58.092 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-02-07T12:46:58.092 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:46:58.092 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr": { 2024-02-07T12:46:58.092 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:46:58.092 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:46:58.092 INFO:teuthology.orchestra.run.smithi019.stdout: "osd": { 2024-02-07T12:46:58.092 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-02-07T12:46:58.093 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:46:58.093 INFO:teuthology.orchestra.run.smithi019.stdout: "mds": {}, 2024-02-07T12:46:58.093 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw": { 2024-02-07T12:46:58.093 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:46:58.093 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:46:58.093 INFO:teuthology.orchestra.run.smithi019.stdout: "overall": { 2024-02-07T12:46:58.093 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-02-07T12:46:58.093 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:46:58.093 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:46:58.093 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:46:58.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:58.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:58.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:46:58.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:46:58.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:46:58.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:46:58.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:46:58.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:46:58.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:46:58.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:46:58.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:46:58.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:46:58.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:46:58.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:46:58.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:46:58.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:46:58.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:46:58.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:46:58.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:46:58.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:58.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:46:58.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:58.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:58.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:58.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:58.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:58.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:46:58.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:58.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:46:58.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:58.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:46:58.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:58.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:58.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:58.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:58.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:58.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:58.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:58.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:58.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:46:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:58.449 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:46:58.449 INFO:teuthology.orchestra.run.smithi019.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", 2024-02-07T12:46:58.449 INFO:teuthology.orchestra.run.smithi019.stdout: "in_progress": true, 2024-02-07T12:46:58.449 INFO:teuthology.orchestra.run.smithi019.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi172", 2024-02-07T12:46:58.450 INFO:teuthology.orchestra.run.smithi019.stdout: "services_complete": [], 2024-02-07T12:46:58.450 INFO:teuthology.orchestra.run.smithi019.stdout: "progress": "", 2024-02-07T12:46:58.450 INFO:teuthology.orchestra.run.smithi019.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 image", 2024-02-07T12:46:58.450 INFO:teuthology.orchestra.run.smithi019.stdout: "is_paused": false 2024-02-07T12:46:58.450 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:46:58.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:58 smithi019 ceph-mon[24574]: from='client.15555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:58.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:58 smithi019 ceph-mon[24574]: from='client.15561 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:58.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:58 smithi019 ceph-mon[24574]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:58.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:46:58.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:58.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:58.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:58.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:58.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:58.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:46:58.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:58.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:46:58.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:58.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:46:58.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:58.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:58.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:58.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:58.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:58.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:58.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:58.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:58.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:58.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:46:58.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:46:58.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:46:58.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:46:58.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:46:58.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:46:58.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:46:58.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:46:58.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:46:58.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:46:58.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:46:58.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:46:58.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:46:58.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:46:58.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:46:58.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:46:58.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:46:58.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:58.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:46:58.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:58.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:58.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:58.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:58.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:58.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:46:58.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:58.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:46:58.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:58.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:46:58.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:58.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:58.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:58.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:58.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:58.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:58.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:58.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:58.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:58.599 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:58 smithi019 ceph-mon[24574]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:46:58.599 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:58 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/2765024211' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:58.599 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:58 smithi019 ceph-mon[29267]: from='client.15555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:58.599 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:58 smithi019 ceph-mon[29267]: from='client.15561 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:58.599 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:58 smithi019 ceph-mon[29267]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:58.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:46:58.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:58.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:58.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:58.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:58.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:58.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:46:58.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:58.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:46:58.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:58.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:46:58.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:58.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:58.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:58.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:58.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:58.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:58.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:58.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:58.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:58.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:46:58.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:46:58.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:46:58.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:46:58.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:46:58.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:46:58.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:46:58.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:46:58.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:46:58.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:46:58.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:46:58.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:46:58.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:46:58.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:46:58.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:46:58.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:46:58.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:46:58.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:58.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:46:58.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:58.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:58.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:58.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:58.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:58.605 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:46:58.605 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:58.605 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:46:58.605 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:58.605 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:46:58.605 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:58.605 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:58.605 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:58.605 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:58.605 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:58.605 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:58.605 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:58.605 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:58.605 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:58.606 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:58 smithi019 ceph-mon[29267]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:46:58.606 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:58 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/2765024211' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:58.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:58 smithi172 ceph-mon[25457]: from='client.15555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:58.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:58 smithi172 ceph-mon[25457]: from='client.15561 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:58.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:58 smithi172 ceph-mon[25457]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:58.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:46:58.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:58.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:58.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:58.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:58.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:58.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:46:58.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:58.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:46:58.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:58.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:46:58.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:58.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:58.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:58.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:58.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:58.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:58.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:58.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:58.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:58.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:46:58.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:46:58.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:46:58.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:46:58.625 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:46:58.625 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:46:58.625 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:46:58.625 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:46:58.625 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:46:58.625 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:46:58.626 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:46:58.626 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:46:58.626 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:46:58.626 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:46:58.626 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:46:58.626 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:46:58.626 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:46:58.627 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:46:58.627 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:46:58.627 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:46:58.627 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:46:58.627 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:46:58.627 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:58.628 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:58.628 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:46:58.628 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:46:58.628 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:46:58.628 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:46:58.628 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:46:58.628 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:46:58.629 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:46:58.629 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:46:58.629 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:46:58.629 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:46:58.629 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:46:58.629 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:46:58.630 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:46:58.630 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:46:58.630 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:58 smithi172 ceph-mon[25457]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:46:58.630 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:58 smithi172 ceph-mon[25457]: from='client.? 172.21.15.19:0/2765024211' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:46:59.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:59 smithi019 ceph-mon[29267]: from='client.15573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:59.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:46:59 smithi019 ceph-mon[29267]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:46:59.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:59 smithi019 ceph-mon[24574]: from='client.15573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:59.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:46:59 smithi019 ceph-mon[24574]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:46:59.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:59 smithi172 ceph-mon[25457]: from='client.15573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:46:59.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:46:59 smithi172 ceph-mon[25457]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:47:01.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:00 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:01.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:00 smithi019 ceph-mon[24574]: Upgrade: Target is version 19.0.0-1252-g44458cce (squid) 2024-02-07T12:47:01.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:00 smithi019 ceph-mon[24574]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe'] 2024-02-07T12:47:01.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:00 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:47:01.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:00 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:01.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:00 smithi019 ceph-mon[24574]: Upgrade: Setting container_image for all mgr 2024-02-07T12:47:01.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:00 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:01.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:00 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-02-07T12:47:01.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:00 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-02-07T12:47:01.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:00 smithi019 ceph-mon[24574]: Upgrade: It appears safe to stop mon.b 2024-02-07T12:47:01.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:00 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:47:01.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:00 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:01.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:00 smithi019 ceph-mon[29267]: Upgrade: Target is version 19.0.0-1252-g44458cce (squid) 2024-02-07T12:47:01.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:00 smithi019 ceph-mon[29267]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe'] 2024-02-07T12:47:01.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:00 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:47:01.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:00 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:01.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:00 smithi019 ceph-mon[29267]: Upgrade: Setting container_image for all mgr 2024-02-07T12:47:01.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:00 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:01.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:00 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-02-07T12:47:01.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:00 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-02-07T12:47:01.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:00 smithi019 ceph-mon[29267]: Upgrade: It appears safe to stop mon.b 2024-02-07T12:47:01.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:00 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:47:01.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:00 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:01.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:00 smithi172 ceph-mon[25457]: Upgrade: Target is version 19.0.0-1252-g44458cce (squid) 2024-02-07T12:47:01.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:00 smithi172 ceph-mon[25457]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe'] 2024-02-07T12:47:01.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:00 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:47:01.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:00 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:01.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:00 smithi172 ceph-mon[25457]: Upgrade: Setting container_image for all mgr 2024-02-07T12:47:01.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:00 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:01.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:00 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-02-07T12:47:01.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:00 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-02-07T12:47:01.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:00 smithi172 ceph-mon[25457]: Upgrade: It appears safe to stop mon.b 2024-02-07T12:47:01.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:00 smithi172 ceph-mon[25457]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:47:02.244 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:01 smithi172 ceph-mon[25457]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:47:02.244 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:01 smithi172 ceph-mon[25457]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:02.244 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:01 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:47:02.244 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:01 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:47:02.244 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:01 smithi172 ceph-mon[25457]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:02.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:01 smithi019 ceph-mon[24574]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:47:02.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:01 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:02.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:01 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:47:02.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:01 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:47:02.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:01 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:02.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:01 smithi019 ceph-mon[29267]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:47:02.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:01 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:02.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:01 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:47:02.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:01 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:47:02.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:01 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:03.200 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:02 smithi172 ceph-mon[25457]: Upgrade: Updating mon.b 2024-02-07T12:47:03.200 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:02 smithi172 ceph-mon[25457]: Deploying daemon mon.b on smithi172 2024-02-07T12:47:03.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:02 smithi019 ceph-mon[24574]: Upgrade: Updating mon.b 2024-02-07T12:47:03.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:02 smithi019 ceph-mon[24574]: Deploying daemon mon.b on smithi172 2024-02-07T12:47:03.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:02 smithi019 ceph-mon[29267]: Upgrade: Updating mon.b 2024-02-07T12:47:03.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:02 smithi019 ceph-mon[29267]: Deploying daemon mon.b on smithi172 2024-02-07T12:47:03.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:03 smithi172 systemd[1]: Stopping Ceph mon.b for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:47:03.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:03 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-b[25442]: 2024-02-07T12:47:03.718+0000 7fb7c72d9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:47:03.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:03 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-b[25442]: 2024-02-07T12:47:03.718+0000 7fb7c72d9700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-02-07T12:47:04.205 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:03 smithi172 podman[67472]: 2024-02-07 12:47:03.913563536 +0000 UTC m=+0.288096236 container died 7d62c7c4d075b789ea2a026b6848c89c9273a14590b88e7050e7e1d8a37ff299 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-b, io.buildah.version=1.19.8, release=754, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , io.openshift.expose-services=, distribution-scope=public, name=centos-stream, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, version=8, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-02-07T12:47:04.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:47:03] "GET /metrics HTTP/1.1" 200 37994 "" "Prometheus/2.43.0" 2024-02-07T12:47:04.530 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:04 smithi172 podman[67472]: 2024-02-07 12:47:04.294119497 +0000 UTC m=+0.668652191 container cleanup 7d62c7c4d075b789ea2a026b6848c89c9273a14590b88e7050e7e1d8a37ff299 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-b, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, vcs-type=git, vendor=Red Hat, Inc., distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , architecture=x86_64, release=754, RELEASE=HEAD) 2024-02-07T12:47:04.530 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:04 smithi172 bash[67472]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-b 2024-02-07T12:47:04.842 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:04 smithi172 podman[67487]: 2024-02-07 12:47:04.529005389 +0000 UTC m=+0.616248377 container remove 7d62c7c4d075b789ea2a026b6848c89c9273a14590b88e7050e7e1d8a37ff299 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-b, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, name=centos-stream, io.buildah.version=1.19.8, ceph=True, GIT_CLEAN=True, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, maintainer=Guillaume Abrioux , vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-02-07T12:47:04.842 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:04 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mon.b.service: Deactivated successfully. 2024-02-07T12:47:04.842 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:04 smithi172 systemd[1]: Stopped Ceph mon.b for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:47:04.842 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:04 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mon.b.service: Consumed 17.770s CPU time. 2024-02-07T12:47:04.843 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:04 smithi172 systemd[1]: Starting Ceph mon.b for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:47:05.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 podman[67617]: 2024-02-07 12:47:04.9606848 +0000 UTC m=+0.019150333 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:47:05.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 podman[67617]: 2024-02-07 12:47:05.088680446 +0000 UTC m=+0.147145970 container create 00f454bbac01aa3cd5db09cb8e50ae2a5d08c2577d65b3af7715ee8d4d7ef6ac (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-b, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:47:05.577 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 podman[67617]: 2024-02-07 12:47:05.291809362 +0000 UTC m=+0.350274888 container init 00f454bbac01aa3cd5db09cb8e50ae2a5d08c2577d65b3af7715ee8d4d7ef6ac (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-b, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:47:05.577 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 podman[67617]: 2024-02-07 12:47:05.294878227 +0000 UTC m=+0.353343755 container start 00f454bbac01aa3cd5db09cb8e50ae2a5d08c2577d65b3af7715ee8d4d7ef6ac (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-b, io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131, org.label-schema.license=GPLv2) 2024-02-07T12:47:05.578 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: set uid:gid to 167:167 (ceph:ceph) 2024-02-07T12:47:05.578 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev), process ceph-mon, pid 2 2024-02-07T12:47:05.578 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: pidfile_write: ignore empty --pid-file 2024-02-07T12:47:05.578 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: load: jerasure load: lrc 2024-02-07T12:47:05.578 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: RocksDB version: 7.9.2 2024-02-07T12:47:05.578 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Git sha 0 2024-02-07T12:47:05.578 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Compile date 2024-02-05 19:41:51 2024-02-07T12:47:05.578 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: DB SUMMARY 2024-02-07T12:47:05.578 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: DB Session ID: M7TGCJ27E5SBQIM1I40Q 2024-02-07T12:47:05.578 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: CURRENT file: CURRENT 2024-02-07T12:47:05.578 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: IDENTITY file: IDENTITY 2024-02-07T12:47:05.578 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: MANIFEST file: MANIFEST-000009 size: 2814 Bytes 2024-02-07T12:47:05.578 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000054.sst 2024-02-07T12:47:05.578 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000052.log size: 2012650 ; 2024-02-07T12:47:05.579 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.error_if_exists: 0 2024-02-07T12:47:05.579 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.create_if_missing: 0 2024-02-07T12:47:05.579 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.paranoid_checks: 1 2024-02-07T12:47:05.579 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.flush_verify_memtable_count: 1 2024-02-07T12:47:05.579 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-02-07T12:47:05.579 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-02-07T12:47:05.579 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.env: 0x558489c91ae0 2024-02-07T12:47:05.579 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.fs: PosixFileSystem 2024-02-07T12:47:05.579 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.info_log: 0x55848be118c0 2024-02-07T12:47:05.579 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.max_file_opening_threads: 16 2024-02-07T12:47:05.580 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.statistics: (nil) 2024-02-07T12:47:05.580 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.use_fsync: 0 2024-02-07T12:47:05.580 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.max_log_file_size: 0 2024-02-07T12:47:05.581 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-02-07T12:47:05.581 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.log_file_time_to_roll: 0 2024-02-07T12:47:05.581 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.keep_log_file_num: 1000 2024-02-07T12:47:05.581 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.recycle_log_file_num: 0 2024-02-07T12:47:05.581 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.allow_fallocate: 1 2024-02-07T12:47:05.581 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.allow_mmap_reads: 0 2024-02-07T12:47:05.581 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.allow_mmap_writes: 0 2024-02-07T12:47:05.581 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.use_direct_reads: 0 2024-02-07T12:47:05.581 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-02-07T12:47:05.581 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.create_missing_column_families: 0 2024-02-07T12:47:05.581 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.db_log_dir: 2024-02-07T12:47:05.581 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.wal_dir: 2024-02-07T12:47:05.581 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.table_cache_numshardbits: 6 2024-02-07T12:47:05.581 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.WAL_ttl_seconds: 0 2024-02-07T12:47:05.581 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.WAL_size_limit_MB: 0 2024-02-07T12:47:05.582 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-02-07T12:47:05.582 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-02-07T12:47:05.582 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.is_fd_close_on_exec: 1 2024-02-07T12:47:05.582 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.advise_random_on_open: 1 2024-02-07T12:47:05.582 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.db_write_buffer_size: 0 2024-02-07T12:47:05.582 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.write_buffer_manager: 0x55848bde4dc0 2024-02-07T12:47:05.582 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-02-07T12:47:05.582 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-02-07T12:47:05.582 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.use_adaptive_mutex: 0 2024-02-07T12:47:05.582 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.rate_limiter: (nil) 2024-02-07T12:47:05.582 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-02-07T12:47:05.582 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.wal_recovery_mode: 2 2024-02-07T12:47:05.582 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.enable_thread_tracking: 0 2024-02-07T12:47:05.582 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.enable_pipelined_write: 0 2024-02-07T12:47:05.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.unordered_write: 0 2024-02-07T12:47:05.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-02-07T12:47:05.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-02-07T12:47:05.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-02-07T12:47:05.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-02-07T12:47:05.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.row_cache: None 2024-02-07T12:47:05.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.wal_filter: None 2024-02-07T12:47:05.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-02-07T12:47:05.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.allow_ingest_behind: 0 2024-02-07T12:47:05.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.two_write_queues: 0 2024-02-07T12:47:05.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.manual_wal_flush: 0 2024-02-07T12:47:05.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.wal_compression: 0 2024-02-07T12:47:05.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.atomic_flush: 0 2024-02-07T12:47:05.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-02-07T12:47:05.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.persist_stats_to_disk: 0 2024-02-07T12:47:05.583 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.write_dbid_to_manifest: 0 2024-02-07T12:47:05.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.log_readahead_size: 0 2024-02-07T12:47:05.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-02-07T12:47:05.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.best_efforts_recovery: 0 2024-02-07T12:47:05.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-02-07T12:47:05.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-02-07T12:47:05.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.allow_data_in_errors: 0 2024-02-07T12:47:05.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.db_host_id: __hostname__ 2024-02-07T12:47:05.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.enforce_single_del_contracts: true 2024-02-07T12:47:05.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.max_background_jobs: 2 2024-02-07T12:47:05.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.max_background_compactions: -1 2024-02-07T12:47:05.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.max_subcompactions: 1 2024-02-07T12:47:05.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-02-07T12:47:05.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-02-07T12:47:05.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.delayed_write_rate : 16777216 2024-02-07T12:47:05.584 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.max_total_wal_size: 0 2024-02-07T12:47:05.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-02-07T12:47:05.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.stats_dump_period_sec: 600 2024-02-07T12:47:05.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.stats_persist_period_sec: 600 2024-02-07T12:47:05.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-02-07T12:47:05.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.max_open_files: -1 2024-02-07T12:47:05.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.bytes_per_sync: 0 2024-02-07T12:47:05.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.wal_bytes_per_sync: 0 2024-02-07T12:47:05.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.strict_bytes_per_sync: 0 2024-02-07T12:47:05.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.compaction_readahead_size: 0 2024-02-07T12:47:05.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.max_background_flushes: -1 2024-02-07T12:47:05.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Compression algorithms supported: 2024-02-07T12:47:05.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-02-07T12:47:05.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: kZSTD supported: 0 2024-02-07T12:47:05.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: kXpressCompression supported: 0 2024-02-07T12:47:05.585 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: kLZ4HCCompression supported: 1 2024-02-07T12:47:05.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: kZlibCompression supported: 1 2024-02-07T12:47:05.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: kSnappyCompression supported: 1 2024-02-07T12:47:05.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: kLZ4Compression supported: 1 2024-02-07T12:47:05.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: kBZip2Compression supported: 0 2024-02-07T12:47:05.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Fast CRC32 supported: Supported on x86 2024-02-07T12:47:05.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: DMutex implementation: pthread_mutex_t 2024-02-07T12:47:05.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-02-07T12:47:05.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-02-07T12:47:05.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-02-07T12:47:05.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.merge_operator: 2024-02-07T12:47:05.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.compaction_filter: None 2024-02-07T12:47:05.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.compaction_filter_factory: None 2024-02-07T12:47:05.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.sst_partitioner_factory: None 2024-02-07T12:47:05.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.memtable_factory: SkipListFactory 2024-02-07T12:47:05.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.table_factory: BlockBasedTable 2024-02-07T12:47:05.587 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55848be119c0) 2024-02-07T12:47:05.587 INFO:journalctl@ceph.mon.b.smithi172.stdout: cache_index_and_filter_blocks: 1 2024-02-07T12:47:05.587 INFO:journalctl@ceph.mon.b.smithi172.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-02-07T12:47:05.587 INFO:journalctl@ceph.mon.b.smithi172.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-02-07T12:47:05.587 INFO:journalctl@ceph.mon.b.smithi172.stdout: pin_top_level_index_and_filter: 1 2024-02-07T12:47:05.587 INFO:journalctl@ceph.mon.b.smithi172.stdout: index_type: 0 2024-02-07T12:47:05.587 INFO:journalctl@ceph.mon.b.smithi172.stdout: data_block_index_type: 0 2024-02-07T12:47:05.587 INFO:journalctl@ceph.mon.b.smithi172.stdout: index_shortening: 1 2024-02-07T12:47:05.587 INFO:journalctl@ceph.mon.b.smithi172.stdout: data_block_hash_table_util_ratio: 0.750000 2024-02-07T12:47:05.587 INFO:journalctl@ceph.mon.b.smithi172.stdout: checksum: 4 2024-02-07T12:47:05.587 INFO:journalctl@ceph.mon.b.smithi172.stdout: no_block_cache: 0 2024-02-07T12:47:05.587 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_cache: 0x55848be15090 2024-02-07T12:47:05.587 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_cache_name: BinnedLRUCache 2024-02-07T12:47:05.587 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_cache_options: 2024-02-07T12:47:05.587 INFO:journalctl@ceph.mon.b.smithi172.stdout: capacity : 536870912 2024-02-07T12:47:05.587 INFO:journalctl@ceph.mon.b.smithi172.stdout: num_shard_bits : 4 2024-02-07T12:47:05.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: strict_capacity_limit : 0 2024-02-07T12:47:05.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: high_pri_pool_ratio: 0.000 2024-02-07T12:47:05.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_cache_compressed: (nil) 2024-02-07T12:47:05.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: persistent_cache: (nil) 2024-02-07T12:47:05.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_size: 4096 2024-02-07T12:47:05.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_size_deviation: 10 2024-02-07T12:47:05.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_restart_interval: 16 2024-02-07T12:47:05.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: index_block_restart_interval: 1 2024-02-07T12:47:05.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: metadata_block_size: 4096 2024-02-07T12:47:05.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: partition_filters: 0 2024-02-07T12:47:05.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_delta_encoding: 1 2024-02-07T12:47:05.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: filter_policy: bloomfilter 2024-02-07T12:47:05.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: whole_key_filtering: 1 2024-02-07T12:47:05.588 INFO:journalctl@ceph.mon.b.smithi172.stdout: verify_compression: 0 2024-02-07T12:47:05.589 INFO:journalctl@ceph.mon.b.smithi172.stdout: read_amp_bytes_per_bit: 0 2024-02-07T12:47:05.589 INFO:journalctl@ceph.mon.b.smithi172.stdout: format_version: 5 2024-02-07T12:47:05.589 INFO:journalctl@ceph.mon.b.smithi172.stdout: enable_index_compression: 1 2024-02-07T12:47:05.589 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_align: 0 2024-02-07T12:47:05.589 INFO:journalctl@ceph.mon.b.smithi172.stdout: max_auto_readahead_size: 262144 2024-02-07T12:47:05.589 INFO:journalctl@ceph.mon.b.smithi172.stdout: prepopulate_block_cache: 0 2024-02-07T12:47:05.589 INFO:journalctl@ceph.mon.b.smithi172.stdout: initial_auto_readahead_size: 8192 2024-02-07T12:47:05.589 INFO:journalctl@ceph.mon.b.smithi172.stdout: num_file_reads_for_auto_readahead: 2 2024-02-07T12:47:05.589 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.write_buffer_size: 33554432 2024-02-07T12:47:05.589 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.max_write_buffer_number: 2 2024-02-07T12:47:05.589 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.compression: NoCompression 2024-02-07T12:47:05.589 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.bottommost_compression: Disabled 2024-02-07T12:47:05.589 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.prefix_extractor: nullptr 2024-02-07T12:47:05.589 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-02-07T12:47:05.589 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.num_levels: 7 2024-02-07T12:47:05.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-02-07T12:47:05.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-02-07T12:47:05.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-02-07T12:47:05.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-02-07T12:47:05.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-02-07T12:47:05.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-02-07T12:47:05.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-02-07T12:47:05.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-02-07T12:47:05.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-02-07T12:47:05.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-02-07T12:47:05.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-02-07T12:47:05.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-02-07T12:47:05.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.compression_opts.window_bits: -14 2024-02-07T12:47:05.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.compression_opts.level: 32767 2024-02-07T12:47:05.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.compression_opts.strategy: 0 2024-02-07T12:47:05.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-02-07T12:47:05.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-02-07T12:47:05.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-02-07T12:47:05.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-02-07T12:47:05.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.compression_opts.enabled: false 2024-02-07T12:47:05.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-02-07T12:47:05.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-02-07T12:47:05.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-02-07T12:47:05.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-02-07T12:47:05.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.target_file_size_base: 67108864 2024-02-07T12:47:05.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.target_file_size_multiplier: 1 2024-02-07T12:47:05.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-02-07T12:47:05.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-02-07T12:47:05.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-02-07T12:47:05.591 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-02-07T12:47:05.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-02-07T12:47:05.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-02-07T12:47:05.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-02-07T12:47:05.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-02-07T12:47:05.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-02-07T12:47:05.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-02-07T12:47:05.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-02-07T12:47:05.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-02-07T12:47:05.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-02-07T12:47:05.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.arena_block_size: 1048576 2024-02-07T12:47:05.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-02-07T12:47:05.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-02-07T12:47:05.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.disable_auto_compactions: 0 2024-02-07T12:47:05.592 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-02-07T12:47:05.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-02-07T12:47:05.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-02-07T12:47:05.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-02-07T12:47:05.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-02-07T12:47:05.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-02-07T12:47:05.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-02-07T12:47:05.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-02-07T12:47:05.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-02-07T12:47:05.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-02-07T12:47:05.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.table_properties_collectors: 2024-02-07T12:47:05.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.inplace_update_support: 0 2024-02-07T12:47:05.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.inplace_update_num_locks: 10000 2024-02-07T12:47:05.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-02-07T12:47:05.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-02-07T12:47:05.593 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.memtable_huge_page_size: 0 2024-02-07T12:47:05.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.bloom_locality: 0 2024-02-07T12:47:05.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.max_successive_merges: 0 2024-02-07T12:47:05.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.optimize_filters_for_hits: 0 2024-02-07T12:47:05.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.paranoid_file_checks: 0 2024-02-07T12:47:05.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.force_consistency_checks: 1 2024-02-07T12:47:05.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.report_bg_io_stats: 0 2024-02-07T12:47:05.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.ttl: 2592000 2024-02-07T12:47:05.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.periodic_compaction_seconds: 0 2024-02-07T12:47:05.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-02-07T12:47:05.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-02-07T12:47:05.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.enable_blob_files: false 2024-02-07T12:47:05.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.min_blob_size: 0 2024-02-07T12:47:05.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.blob_file_size: 268435456 2024-02-07T12:47:05.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.blob_compression_type: NoCompression 2024-02-07T12:47:05.594 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.enable_blob_garbage_collection: false 2024-02-07T12:47:05.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-02-07T12:47:05.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-02-07T12:47:05.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-02-07T12:47:05.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.blob_file_starting_level: 0 2024-02-07T12:47:05.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-02-07T12:47:05.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 54.sst 2024-02-07T12:47:05.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-02-07T12:47:05.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 56, last_sequence is 28877, log_number is 52,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-02-07T12:47:05.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 52 2024-02-07T12:47:05.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 4776c340-4345-44aa-954c-47935994e255 2024-02-07T12:47:05.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707310025323924, "job": 1, "event": "recovery_started", "wal_files": [52]} 2024-02-07T12:47:05.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #52 mode 2 2024-02-07T12:47:05.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707310025329592, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 57, "file_size": 1278741, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 28882, "largest_seqno": 30177, "table_properties": {"data_size": 1273383, "index_size": 2999, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1349, "raw_key_size": 12947, "raw_average_key_size": 24, "raw_value_size": 1262472, "raw_average_value_size": 2404, "num_data_blocks": 133, "num_entries": 525, "num_filter_entries": 525, "num_deletions": 7, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1707310025, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "4776c340-4345-44aa-954c-47935994e255", "db_session_id": "M7TGCJ27E5SBQIM1I40Q", "orig_file_number": 57, "seqno_to_time_mapping": "N/A"}} 2024-02-07T12:47:05.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707310025329684, "job": 1, "event": "recovery_finished"} 2024-02-07T12:47:05.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: [db/version_set.cc:5047] Creating manifest 59 2024-02-07T12:47:05.596 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-02-07T12:47:05.596 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000052.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-02-07T12:47:05.596 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55848bf06000 2024-02-07T12:47:05.596 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: DB pointer 0x55848bef0000 2024-02-07T12:47:05.596 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-02-07T12:47:05.596 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-02-07T12:47:05.596 INFO:journalctl@ceph.mon.b.smithi172.stdout: ** DB Stats ** 2024-02-07T12:47:05.596 INFO:journalctl@ceph.mon.b.smithi172.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-02-07T12:47:05.596 INFO:journalctl@ceph.mon.b.smithi172.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-02-07T12:47:05.596 INFO:journalctl@ceph.mon.b.smithi172.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-02-07T12:47:05.596 INFO:journalctl@ceph.mon.b.smithi172.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-07T12:47:05.596 INFO:journalctl@ceph.mon.b.smithi172.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-02-07T12:47:05.596 INFO:journalctl@ceph.mon.b.smithi172.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-02-07T12:47:05.596 INFO:journalctl@ceph.mon.b.smithi172.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-07T12:47:05.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: 2024-02-07T12:47:05.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: ** Compaction Stats [default] ** 2024-02-07T12:47:05.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-02-07T12:47:05.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-02-07T12:47:05.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: L0 1/0 1.22 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 461.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-02-07T12:47:05.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: L6 1/0 8.81 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-02-07T12:47:05.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: Sum 2/0 10.03 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 461.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-02-07T12:47:05.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 461.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-02-07T12:47:05.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: 2024-02-07T12:47:05.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: ** Compaction Stats [default] ** 2024-02-07T12:47:05.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-02-07T12:47:05.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-07T12:47:05.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 461.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-02-07T12:47:05.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: 2024-02-07T12:47:05.597 INFO:journalctl@ceph.mon.b.smithi172.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-02-07T12:47:05.598 INFO:journalctl@ceph.mon.b.smithi172.stdout: 2024-02-07T12:47:05.598 INFO:journalctl@ceph.mon.b.smithi172.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-02-07T12:47:05.598 INFO:journalctl@ceph.mon.b.smithi172.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-02-07T12:47:05.598 INFO:journalctl@ceph.mon.b.smithi172.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-02-07T12:47:05.598 INFO:journalctl@ceph.mon.b.smithi172.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-02-07T12:47:05.598 INFO:journalctl@ceph.mon.b.smithi172.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-02-07T12:47:05.598 INFO:journalctl@ceph.mon.b.smithi172.stdout: AddFile(Keys): cumulative 0, interval 0 2024-02-07T12:47:05.598 INFO:journalctl@ceph.mon.b.smithi172.stdout: Cumulative compaction: 0.00 GB write, 152.84 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:47:05.598 INFO:journalctl@ceph.mon.b.smithi172.stdout: Interval compaction: 0.00 GB write, 152.84 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:47:05.598 INFO:journalctl@ceph.mon.b.smithi172.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-02-07T12:47:05.598 INFO:journalctl@ceph.mon.b.smithi172.stdout: Block cache BinnedLRUCache@0x55848be15090#2 capacity: 512.00 MB usage: 4.48 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-02-07T12:47:05.599 INFO:journalctl@ceph.mon.b.smithi172.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.41 KB,0.000268221%) IndexBlock(1,3.08 KB,0.000587106%) Misc(1,0.00 KB,0%) 2024-02-07T12:47:05.599 INFO:journalctl@ceph.mon.b.smithi172.stdout: 2024-02-07T12:47:05.599 INFO:journalctl@ceph.mon.b.smithi172.stdout: ** File Read Latency Histogram By Level [default] ** 2024-02-07T12:47:05.599 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: starting mon.b rank 2 at public addrs [v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0] at bind addrs [v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:47:05.599 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: mon.b@-1(???) e3 preinit fsid ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:47:05.599 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: mon.b@-1(???).mds e1 new map 2024-02-07T12:47:05.599 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: mon.b@-1(???).mds e1 print_map 2024-02-07T12:47:05.599 INFO:journalctl@ceph.mon.b.smithi172.stdout: e1 2024-02-07T12:47:05.599 INFO:journalctl@ceph.mon.b.smithi172.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-02-07T12:47:05.599 INFO:journalctl@ceph.mon.b.smithi172.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-02-07T12:47:05.599 INFO:journalctl@ceph.mon.b.smithi172.stdout: legacy client fscid: -1 2024-02-07T12:47:05.599 INFO:journalctl@ceph.mon.b.smithi172.stdout: 2024-02-07T12:47:05.599 INFO:journalctl@ceph.mon.b.smithi172.stdout: No filesystems configured 2024-02-07T12:47:05.599 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: mon.b@-1(???).osd e104 crush map has features 3314933000854323200, adjusting msgr requires 2024-02-07T12:47:05.599 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: mon.b@-1(???).osd e104 crush map has features 432629239337189376, adjusting msgr requires 2024-02-07T12:47:05.600 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: mon.b@-1(???).osd e104 crush map has features 432629239337189376, adjusting msgr requires 2024-02-07T12:47:05.600 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: mon.b@-1(???).osd e104 crush map has features 432629239337189376, adjusting msgr requires 2024-02-07T12:47:05.600 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: mon.b@-1(???).paxosservice(auth 1..25) refresh upgraded, format 0 -> 3 2024-02-07T12:47:05.600 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 bash[67617]: 00f454bbac01aa3cd5db09cb8e50ae2a5d08c2577d65b3af7715ee8d4d7ef6ac 2024-02-07T12:47:05.600 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 systemd[1]: Started Ceph mon.b for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:47:05.600 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:05 smithi172 ceph-mon[67636]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:47:05.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:05 smithi019 ceph-mon[29267]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:47:05.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:05 smithi019 ceph-mon[24574]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:47:06.747 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:06 smithi172 ceph-mon[67636]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:47:06.747 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:06 smithi172 ceph-mon[67636]: mon.b calling monitor election 2024-02-07T12:47:06.747 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:06 smithi172 ceph-mon[67636]: mon.c calling monitor election 2024-02-07T12:47:06.747 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:06 smithi172 ceph-mon[67636]: mon.a calling monitor election 2024-02-07T12:47:06.747 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:06 smithi172 ceph-mon[67636]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-02-07T12:47:06.747 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:06 smithi172 ceph-mon[67636]: monmap e3: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} 2024-02-07T12:47:06.747 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:06 smithi172 ceph-mon[67636]: fsmap 2024-02-07T12:47:06.747 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:06 smithi172 ceph-mon[67636]: osdmap e104: 8 total, 8 up, 8 in 2024-02-07T12:47:06.747 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:06 smithi172 ceph-mon[67636]: mgrmap e43: y(active, since 2m), standbys: x 2024-02-07T12:47:06.747 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:06 smithi172 ceph-mon[67636]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-02-07T12:47:06.747 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:06 smithi172 ceph-mon[67636]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-02-07T12:47:06.748 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:06 smithi172 ceph-mon[67636]: daemon iscsi.foo.smithi019.svjimj on smithi019 is in unknown state 2024-02-07T12:47:06.748 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:06 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:06.748 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:06 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:06.748 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:06 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:47:06.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[29267]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:47:06.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[29267]: mon.b calling monitor election 2024-02-07T12:47:06.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[29267]: mon.c calling monitor election 2024-02-07T12:47:06.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[29267]: mon.a calling monitor election 2024-02-07T12:47:06.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[29267]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-02-07T12:47:06.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[29267]: monmap e3: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} 2024-02-07T12:47:06.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[29267]: fsmap 2024-02-07T12:47:06.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[29267]: osdmap e104: 8 total, 8 up, 8 in 2024-02-07T12:47:06.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[29267]: mgrmap e43: y(active, since 2m), standbys: x 2024-02-07T12:47:06.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[29267]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-02-07T12:47:06.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[29267]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-02-07T12:47:06.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[29267]: daemon iscsi.foo.smithi019.svjimj on smithi019 is in unknown state 2024-02-07T12:47:06.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:06.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:06.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:47:06.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[24574]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:47:06.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[24574]: mon.b calling monitor election 2024-02-07T12:47:06.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[24574]: mon.c calling monitor election 2024-02-07T12:47:06.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[24574]: mon.a calling monitor election 2024-02-07T12:47:06.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[24574]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-02-07T12:47:06.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[24574]: monmap e3: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} 2024-02-07T12:47:06.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[24574]: fsmap 2024-02-07T12:47:06.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[24574]: osdmap e104: 8 total, 8 up, 8 in 2024-02-07T12:47:06.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[24574]: mgrmap e43: y(active, since 2m), standbys: x 2024-02-07T12:47:06.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[24574]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-02-07T12:47:06.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[24574]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-02-07T12:47:06.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[24574]: daemon iscsi.foo.smithi019.svjimj on smithi019 is in unknown state 2024-02-07T12:47:06.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:06.847 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:06.847 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:06 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:47:07.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:07 smithi019 ceph-mon[24574]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:47:07.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:07 smithi019 ceph-mon[29267]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:47:07.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:07 smithi172 ceph-mon[67636]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:47:09.581 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:09 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:47:09.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:09 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:47:09.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:09 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:47:10.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:47:10.336+0000 7eff63d62700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-02-07T12:47:10.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:10 smithi019 ceph-mon[24574]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:47:10.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:10 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:10.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:10 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:10.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:10 smithi019 ceph-mon[29267]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:47:10.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:10 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:10.597 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:10 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:10.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:10 smithi172 ceph-mon[67636]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:47:10.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:10 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:10.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:10 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:11.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:11 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:47:11.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:11 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:11.590 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:11 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:11.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:11 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:47:11.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:11.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:11 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:11.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:11 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:47:11.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:11.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:11 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:12.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:12 smithi019 ceph-mon[29267]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:47:12.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:12 smithi019 ceph-mon[24574]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:47:12.615 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:12 smithi172 ceph-mon[67636]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:47:14.254 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:47:13] "GET /metrics HTTP/1.1" 200 37938 "" "Prometheus/2.43.0" 2024-02-07T12:47:14.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:14 smithi172 ceph-mon[67636]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:47:14.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:14 smithi019 ceph-mon[29267]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:47:14.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:14 smithi019 ceph-mon[24574]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:47:15.501 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:15 smithi019 ceph-mon[29267]: Detected new or changed devices on smithi172 2024-02-07T12:47:15.502 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:15 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:15.502 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:15 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:15.502 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:15 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:15.502 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:15 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:47:15.502 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:15 smithi019 ceph-mon[29267]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 885 B/s rd, 0 op/s 2024-02-07T12:47:15.502 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:15 smithi019 ceph-mon[29267]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:47:15.502 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:15 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:15.502 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:15 smithi019 ceph-mon[29267]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:47:15.502 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:15 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:47:15.502 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:15 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:47:15.502 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:15 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:15.502 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:15 smithi019 ceph-mon[29267]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:47:15.503 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:15 smithi019 ceph-mon[24574]: Detected new or changed devices on smithi172 2024-02-07T12:47:15.503 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:15 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:15.503 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:15 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:15.503 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:15 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:15.503 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:15 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:47:15.503 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:15 smithi019 ceph-mon[24574]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 885 B/s rd, 0 op/s 2024-02-07T12:47:15.503 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:15 smithi019 ceph-mon[24574]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:47:15.503 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:15 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:15.503 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:15 smithi019 ceph-mon[24574]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:47:15.503 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:15 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:47:15.503 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:15 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:47:15.503 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:15 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:15.504 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:15 smithi019 ceph-mon[24574]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:47:15.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:15 smithi172 ceph-mon[67636]: Detected new or changed devices on smithi172 2024-02-07T12:47:15.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:15 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:15.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:15 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:15.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:15 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:15.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:15 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:47:15.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:15 smithi172 ceph-mon[67636]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 885 B/s rd, 0 op/s 2024-02-07T12:47:15.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:15 smithi172 ceph-mon[67636]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:47:15.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:15 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:15.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:15 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:47:15.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:15 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:47:15.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:15 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:47:15.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:15 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:15.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:15 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:47:16.480 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:47:16.241+0000 7eff60d5c700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:16.480 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:47:16.480 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:16.480 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:16.481 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:16.481 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:16.481 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:16.481 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:47:16.481 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:16.481 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:47:16.481 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:16.481 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:47:16.481 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:16.481 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:16.481 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:16.481 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:16.481 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:16.481 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:16.481 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:16.482 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:16.482 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:16.482 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:47:16.482 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:47:16.482 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:47:16.482 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:47:16.482 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:47:16.482 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:47:16.482 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:47:16.483 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:47:16.483 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:47:16.483 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:47:16.483 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:47:16.483 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:47:16.483 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:47:16.483 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:47:16.483 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:47:16.483 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:47:16.483 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:47:16.483 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:16.483 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:47:16.483 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:16.483 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:16.483 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:16.484 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:16.484 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:16.484 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:47:16.484 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:16.484 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:47:16.484 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:16.484 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:47:16.484 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:16.484 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:16.484 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:16.484 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:16.484 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:16.484 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:16.484 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:16.485 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:16.485 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:16.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:16.844 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:47:16.844 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:16.844 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:16.844 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:16.844 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:16.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:16.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:47:16.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:16.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:47:16.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:16.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:47:16.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:16.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:16.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:16.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:16.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:16.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:16.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:16.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:16.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:16.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:47:16.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:47:16.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:47:16.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:47:16.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:47:16.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:47:16.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:47:16.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:47:16.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:47:16.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:47:16.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:47:16.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:47:16.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:47:16.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:47:16.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:47:16.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:47:16.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:47:16.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:16.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:47:16.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:16.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:16.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:16.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:16.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:16.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:47:16.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:16.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:47:16.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:16.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:47:16.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:16.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:16.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:16.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:16.848 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:16.848 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:47:16.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:16.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:16.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:16.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:16.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:16.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:47:16.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:16.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:47:16.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:16.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:47:16.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:16.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:16.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:16.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:16.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:16.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:16.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:16.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:16.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:16.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:47:16.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:47:16.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:47:16.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:47:16.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:47:16.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:47:16.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:47:16.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:47:16.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:47:16.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:47:16.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:47:16.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:47:16.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:47:16.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:47:16.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:47:16.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:47:16.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:47:16.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:16.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:47:16.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:16.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:16.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:16.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:16.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:16.852 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:47:16.852 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:16.852 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:47:16.852 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:16.852 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:47:16.852 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:16.852 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:16.852 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:16.852 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:16.853 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:16.853 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:16.853 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:16.853 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:16.853 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:16.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:47:16.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: Upgrade: Setting container_image for all crash 2024-02-07T12:47:16.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:16.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: Upgrade: Setting container_image for all mds 2024-02-07T12:47:16.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:16.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: Upgrade: Setting container_image for all rbd-mirror 2024-02-07T12:47:16.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:16.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: Upgrade: Setting container_image for all cephfs-mirror 2024-02-07T12:47:16.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:16.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-02-07T12:47:16.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: Upgrade: Setting container_image for all ceph-exporter 2024-02-07T12:47:16.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:16.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:16.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.855 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:16.855 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.855 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.855 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.855 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.855 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.855 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.855 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.855 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.855 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.855 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.855 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.855 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.855 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.855 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.855 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:47:16.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:47:16.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-02-07T12:47:16.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-02-07T12:47:16.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:47:16.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:47:16.856 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:16.857 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:16.857 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:16.857 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:16.857 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:16.857 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:47:16.857 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.857 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.857 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: Upgrade: Setting container_image for all crash 2024-02-07T12:47:16.857 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:16.857 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.857 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: Upgrade: Setting container_image for all mds 2024-02-07T12:47:16.857 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:16.857 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.858 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: Upgrade: Setting container_image for all rbd-mirror 2024-02-07T12:47:16.858 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:16.858 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.858 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: Upgrade: Setting container_image for all cephfs-mirror 2024-02-07T12:47:16.858 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:16.858 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.858 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.858 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-02-07T12:47:16.858 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.858 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: Upgrade: Setting container_image for all ceph-exporter 2024-02-07T12:47:16.858 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:16.858 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.858 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:16.858 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.858 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:16.858 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.859 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-02-07T12:47:16.859 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:47:16.859 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:47:16.859 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:47:16.859 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:47:16.859 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-02-07T12:47:16.859 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:47:16.859 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:47:16.859 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:47:16.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:47:16.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-02-07T12:47:16.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:47:16.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:47:16.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-02-07T12:47:16.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:47:16.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:47:16.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:47:16.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:47:16.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-02-07T12:47:16.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:47:16.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:47:16.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-02-07T12:47:16.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:47:16.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:47:16.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-02-07T12:47:16.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:47:16.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:16.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:47:16.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:16.862 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.862 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.863 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.863 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.863 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.863 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.863 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.863 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.863 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.863 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.863 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.863 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.863 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.863 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.863 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.863 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.864 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.864 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:47:16.864 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:47:16.864 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-02-07T12:47:16.864 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.864 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.864 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-02-07T12:47:16.864 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:47:16.864 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:47:16.864 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-02-07T12:47:16.864 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:47:16.864 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:47:16.864 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:47:16.864 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:47:16.864 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-02-07T12:47:16.865 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:47:16.865 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:47:16.865 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:47:16.865 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:47:16.865 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-02-07T12:47:16.865 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.865 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.865 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:47:16.865 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:47:16.865 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-02-07T12:47:16.865 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:47:16.865 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:47:16.865 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:47:16.865 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:47:16.865 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-02-07T12:47:16.866 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:47:16.866 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:47:16.866 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-02-07T12:47:16.866 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.866 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.866 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.866 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.867 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.867 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.867 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.867 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.867 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.867 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.867 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.867 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.867 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:47:16.867 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:47:16.867 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-02-07T12:47:16.867 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:47:16.868 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:16.868 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:47:16.868 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:16 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:16.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:16.869 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:47:16.869 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:16.869 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:16.869 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:16.869 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:16.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:16.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:47:16.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:16.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:47:16.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:16.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:47:16.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:16.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:16.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:16.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:16.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:16.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:16.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:16.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:16.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:16.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:47:16.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:47:16.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:47:16.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:47:16.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:47:16.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:47:16.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:47:16.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:47:16.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:47:16.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:47:16.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:47:16.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:47:16.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:47:16.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:47:16.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:47:16.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:47:16.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:47:16.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:16.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:47:16.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:16.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:16.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:16.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:16.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:16.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:47:16.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:16.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:47:16.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:16.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:47:16.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:16.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:16.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:16.873 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:16.873 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:16.873 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:16.873 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:16.873 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:16.873 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:16.873 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:47:16.873 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.873 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.873 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all crash 2024-02-07T12:47:16.873 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:16.873 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.873 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all mds 2024-02-07T12:47:16.873 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:16.873 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.873 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all rbd-mirror 2024-02-07T12:47:16.874 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:16.874 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.874 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all cephfs-mirror 2024-02-07T12:47:16.874 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:16.874 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.874 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.874 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-02-07T12:47:16.874 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.874 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all ceph-exporter 2024-02-07T12:47:16.874 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:16.874 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.874 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:16.874 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.874 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:16.874 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.875 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.875 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.875 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.875 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.875 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.875 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.875 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.875 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.875 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.875 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.876 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.876 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.876 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.876 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.876 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:16.876 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.876 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-02-07T12:47:16.876 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:47:16.876 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:47:16.876 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-02-07T12:47:16.876 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.876 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.876 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-02-07T12:47:16.876 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:47:16.876 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:47:16.877 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-02-07T12:47:16.877 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:47:16.877 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:47:16.877 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:47:16.877 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:47:16.877 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-02-07T12:47:16.877 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:47:16.877 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:47:16.877 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:47:16.877 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:47:16.877 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-02-07T12:47:16.877 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.877 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.877 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:47:16.877 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:47:16.877 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-02-07T12:47:16.878 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:47:16.878 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:47:16.878 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:47:16.878 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:47:16.878 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-02-07T12:47:16.878 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:47:16.878 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:47:16.878 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-02-07T12:47:16.878 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.878 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.878 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.878 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.878 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.878 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.878 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.878 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.879 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.879 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.879 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.879 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:47:16.879 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:47:16.879 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:47:16.879 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-02-07T12:47:16.879 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:47:16.879 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:16.879 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:47:16.879 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:16 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:17.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[29267]: Upgrade: Setting container_image for all nfs 2024-02-07T12:47:17.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[29267]: Upgrade: Setting container_image for all nvmeof 2024-02-07T12:47:17.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[29267]: Upgrade: Setting container_image for all node-exporter 2024-02-07T12:47:17.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[29267]: Upgrade: Setting container_image for all prometheus 2024-02-07T12:47:17.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[29267]: Upgrade: Setting container_image for all alertmanager 2024-02-07T12:47:17.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[29267]: Upgrade: Setting container_image for all grafana 2024-02-07T12:47:17.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[29267]: Upgrade: Setting container_image for all loki 2024-02-07T12:47:17.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[29267]: Upgrade: Setting container_image for all promtail 2024-02-07T12:47:17.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[29267]: Upgrade: Finalizing container_image settings 2024-02-07T12:47:17.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[29267]: Upgrade: Complete! 2024-02-07T12:47:17.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[29267]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:47:17.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[29267]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-02-07T12:47:17.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[29267]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:47:17.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:47:17.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:47:17.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:17.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[29267]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:47:17.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[24574]: Upgrade: Setting container_image for all nfs 2024-02-07T12:47:17.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[24574]: Upgrade: Setting container_image for all nvmeof 2024-02-07T12:47:17.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[24574]: Upgrade: Setting container_image for all node-exporter 2024-02-07T12:47:17.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[24574]: Upgrade: Setting container_image for all prometheus 2024-02-07T12:47:17.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[24574]: Upgrade: Setting container_image for all alertmanager 2024-02-07T12:47:17.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[24574]: Upgrade: Setting container_image for all grafana 2024-02-07T12:47:17.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[24574]: Upgrade: Setting container_image for all loki 2024-02-07T12:47:17.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[24574]: Upgrade: Setting container_image for all promtail 2024-02-07T12:47:17.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[24574]: Upgrade: Finalizing container_image settings 2024-02-07T12:47:17.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[24574]: Upgrade: Complete! 2024-02-07T12:47:17.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[24574]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:47:17.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[24574]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-02-07T12:47:17.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[24574]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:47:17.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:47:17.847 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:47:17.847 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:17.847 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:17 smithi019 ceph-mon[24574]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:47:17.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:17 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all nfs 2024-02-07T12:47:17.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:17 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all nvmeof 2024-02-07T12:47:17.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:17 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all node-exporter 2024-02-07T12:47:17.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:17 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all prometheus 2024-02-07T12:47:17.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:17 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all alertmanager 2024-02-07T12:47:17.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:17 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all grafana 2024-02-07T12:47:17.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:17 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all loki 2024-02-07T12:47:17.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:17 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all promtail 2024-02-07T12:47:17.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:17 smithi172 ceph-mon[67636]: Upgrade: Finalizing container_image settings 2024-02-07T12:47:17.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:17 smithi172 ceph-mon[67636]: Upgrade: Complete! 2024-02-07T12:47:17.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:17 smithi172 ceph-mon[67636]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:47:17.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:17 smithi172 ceph-mon[67636]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-02-07T12:47:17.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:17 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:47:17.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:17 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:47:17.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:17 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:47:17.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:17 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:17.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:17 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:47:18.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:47:18.005+0000 7eff60d5c700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:18.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:47:18.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:18.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:18.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:18.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:18.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:18.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:47:18.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:18.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:47:18.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:18.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:47:18.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:18.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:18.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:18.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:18.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:18.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:18.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:18.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:18.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:18.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:47:18.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:47:18.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:47:18.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:47:18.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:47:18.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:47:18.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:47:18.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:47:18.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:47:18.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:47:18.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:47:18.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:47:18.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:47:18.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:47:18.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:47:18.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:47:18.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:47:18.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:18.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:47:18.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:18.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:18.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:18.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:18.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:18.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:47:18.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:18.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:47:18.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:18.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:47:18.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:18.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:18.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:18.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:18.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:18.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:18.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:18.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:18.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:18.726 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-mon[29267]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:18.726 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:47:18.726 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:18.726 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:18.726 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:18.726 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:18.726 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:18.727 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:47:18.727 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:18.727 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:47:18.727 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:18.727 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:47:18.727 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:18.727 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:18.727 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:18.727 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:18.727 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:18.727 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:18.727 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:18.727 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:18.727 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:18.728 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:47:18.728 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:47:18.728 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:47:18.728 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:47:18.728 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:47:18.728 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:47:18.728 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:47:18.728 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:47:18.728 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:47:18.728 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:47:18.728 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:47:18.728 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:47:18.728 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:47:18.728 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:47:18.728 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:47:18.729 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:47:18.729 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:47:18.729 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:18.729 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:47:18.729 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:18.729 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:18.729 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:18.729 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:18.729 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:18.729 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:47:18.729 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:18.729 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:47:18.729 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:18.729 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:47:18.730 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:18.730 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:18.730 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:18.730 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:18.730 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-mon[24574]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:18.730 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:47:18.730 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:18.730 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:18.730 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:18.730 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:18.731 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:18.731 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:47:18.731 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:18.731 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:47:18.731 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:18.731 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:47:18.731 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:18.731 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:18.731 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:18.731 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:18.731 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:18.731 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:18.731 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:18.731 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:18.732 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:18.732 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:47:18.732 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:47:18.732 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:47:18.732 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:47:18.732 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:47:18.732 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:47:18.732 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:47:18.732 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:47:18.732 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:47:18.732 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:47:18.733 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:47:18.733 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:47:18.733 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:47:18.733 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:47:18.733 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:47:18.733 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:47:18.733 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:47:18.733 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:18.733 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:47:18.733 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:18.733 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:18.733 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:18.733 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:18.734 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:18.734 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:47:18.734 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:18.734 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:47:18.734 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:18.734 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:47:18.734 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:18.734 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:18.734 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:18.734 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:18.734 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:18.735 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:18.735 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:18.735 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:18.735 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:18.735 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:47:18.735 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:18.735 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:47:18.735 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-mon[29267]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 722 B/s rd, 0 op/s 2024-02-07T12:47:18.735 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-mon[29267]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1006 B/s rd, 0 op/s 2024-02-07T12:47:18.735 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:18.735 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-mon[29267]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:47:18.735 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:47:18.735 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:47:18.736 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:18.736 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-mon[29267]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:47:18.736 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:18.736 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:18.736 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:18.736 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:18.736 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:18.736 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:47:18.736 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:18.736 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:47:18.736 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-mon[24574]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 722 B/s rd, 0 op/s 2024-02-07T12:47:18.737 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-mon[24574]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1006 B/s rd, 0 op/s 2024-02-07T12:47:18.737 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:18.737 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-mon[24574]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:47:18.737 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:47:18.737 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:47:18.737 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:18.737 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:18 smithi019 ceph-mon[24574]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:47:18.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:18 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:18.869 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:47:18.869 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:18.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:18.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:18.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:18.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:18.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:47:18.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:18.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:47:18.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:18.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:47:18.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:18.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:18.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:18.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:18.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:18.870 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:18.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:18.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:18.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:18.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:47:18.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:47:18.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:47:18.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:47:18.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:47:18.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:47:18.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:47:18.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:47:18.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:47:18.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:47:18.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:47:18.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:47:18.871 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:47:18.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:47:18.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:47:18.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:47:18.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:47:18.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:18.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:47:18.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:18.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:18.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:18.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:18.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:18.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:47:18.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:18.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:47:18.872 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:18.873 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:47:18.873 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:18.873 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:18.873 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:18.873 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:18.873 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:18.873 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:18.873 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:18.873 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:18.873 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:18.873 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:18 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:47:18.873 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:18 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:18.873 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:18 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:47:18.873 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:18 smithi172 ceph-mon[67636]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 722 B/s rd, 0 op/s 2024-02-07T12:47:18.873 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:18 smithi172 ceph-mon[67636]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1006 B/s rd, 0 op/s 2024-02-07T12:47:18.874 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:18 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:18.874 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:18 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:47:18.874 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:18 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:47:18.874 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:18 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:47:18.874 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:18 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:18.874 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:18 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:47:20.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:47:19.788+0000 7eff60d5c700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:20.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:47:20.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:20.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:20.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:20.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:20.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:20.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:47:20.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:20.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:47:20.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:20.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:47:20.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:20.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:20.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:20.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:20.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:20.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:20.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:20.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:20.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:20.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:47:20.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:47:20.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:47:20.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:47:20.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:47:20.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:47:20.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:47:20.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:47:20.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:47:20.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:47:20.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:47:20.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:47:20.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:47:20.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:47:20.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:47:20.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:47:20.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:47:20.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:20.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:47:20.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:20.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:20.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:20.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:20.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:20.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:47:20.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:20.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:47:20.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:20.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:47:20.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:20.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:20.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:20.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:20.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:20.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:20.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:20.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:20.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:19 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:20.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:20 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:20.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:20 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:20.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:20 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:21 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:47:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:47:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:47:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:47:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:21.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:21.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:21.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:21.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:21.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:21.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:21.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:21.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:47:21.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:47:21.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:47:21.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:47:21.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:47:21.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:47:21.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:47:21.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:47:21.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:47:21.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:47:21.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:47:21.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:47:21.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:47:21.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:47:21.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:47:21.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:47:21.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:47:21.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:21.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:47:21.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:21.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:21.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:21.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:21.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:21.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:47:21.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:21.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:47:21.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:21.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:47:21.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:21.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:21.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:21.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:21.374 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:21.374 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:21.374 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:21.374 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:21.374 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:21.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:21 smithi172 ceph-mon[67636]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-02-07T12:47:21.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:21 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:47:21.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:21 smithi019 ceph-mon[29267]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:21.594 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:47:21.594 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:21.594 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:21.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:21.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:21.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:21.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:47:21.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:21.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:47:21.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:21.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:47:21.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:21.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:21.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:21.595 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:21.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:21.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:21.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:21.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:21.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:21.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:47:21.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:47:21.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:47:21.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:47:21.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:47:21.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:47:21.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:47:21.596 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:47:21.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:47:21.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:47:21.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:47:21.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:47:21.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:47:21.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:47:21.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:47:21.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:47:21.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:47:21.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:21.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:47:21.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:21.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:21.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:21.597 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:21.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:21.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:47:21.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:21.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:47:21.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:21.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:47:21.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:21.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:21.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:21.598 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:21.599 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:21 smithi019 ceph-mon[24574]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:21.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:47:21.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:21.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:21.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:21.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:21.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:21.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:47:21.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:21.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:47:21.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:21.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:47:21.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:21.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:21.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:21.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:21.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:21.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:21.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:21.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:21.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:21.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:47:21.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:47:21.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:47:21.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:47:21.600 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:47:21.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:47:21.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:47:21.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:47:21.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:47:21.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:47:21.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:47:21.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:47:21.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:47:21.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:47:21.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:47:21.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:47:21.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:47:21.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:21.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:47:21.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:21.601 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:21.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:21.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:21.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:21.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:47:21.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:21.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:47:21.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:21.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:47:21.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:21.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:21.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:21.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:21.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:21.602 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:21.603 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:21.603 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:21.603 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:21.603 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:21 smithi019 ceph-mon[24574]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-02-07T12:47:21.603 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:21 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:47:21.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:21.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:21.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:21.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:21.604 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:21.604 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:21 smithi019 ceph-mon[29267]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-02-07T12:47:21.604 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:21 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:47:23.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:23 smithi172 ceph-mon[67636]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-02-07T12:47:23.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:23 smithi019 ceph-mon[29267]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-02-07T12:47:23.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:23 smithi019 ceph-mon[24574]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-02-07T12:47:24.247 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:47:23] "GET /metrics HTTP/1.1" 200 37938 "" "Prometheus/2.43.0" 2024-02-07T12:47:24.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:24 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:47:24.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:24 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:47:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:24 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:47:25.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:25 smithi019 ceph-mon[29267]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 670 B/s rd, 0 op/s 2024-02-07T12:47:25.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:25 smithi019 ceph-mon[24574]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 670 B/s rd, 0 op/s 2024-02-07T12:47:25.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:25 smithi172 ceph-mon[67636]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 670 B/s rd, 0 op/s 2024-02-07T12:47:27.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:27 smithi019 ceph-mon[29267]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:47:27.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:27 smithi019 ceph-mon[24574]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:47:27.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:27 smithi172 ceph-mon[67636]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:47:29.398 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:29 smithi019 ceph-mon[24574]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:47:29.398 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:29 smithi019 ceph-mon[29267]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:47:29.425 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch ps' 2024-02-07T12:47:29.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:29 smithi172 ceph-mon[67636]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:47:30.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:30 smithi019 ceph-mon[29267]: from='client.15579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:47:30.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:30 smithi019 ceph-mon[24574]: from='client.15579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:47:30.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:30 smithi172 ceph-mon[67636]: from='client.15579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:47:31.396 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:47:31.396 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (108s) 103s ago 23m 17.3M - 0.25.0 c8568f914cd2 fec9255f35e7 2024-02-07T12:47:31.396 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (91s) 21s ago 23m 80.1M - 9.4.12 a0ea5e3c46d9 aaffac63bc28 2024-02-07T12:47:31.397 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 unknown 103s ago 22m 57.2M - 3.5 e1d6a67b021e 5113aa77e3f2 2024-02-07T12:47:31.397 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283,8765 running (2m) 21s ago 28m 452M - 19.0.0-1252-g44458cce 79957b372307 5aa0c04fdfd7 2024-02-07T12:47:31.397 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (12m) 103s ago 29m 503M - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:47:31.397 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (29m) 103s ago 29m 79.6M 2048M 17.2.0 e1d6a67b021e f3a8a050cf1d 2024-02-07T12:47:31.397 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (26s) 21s ago 28m 20.4M 2048M 19.0.0-1252-g44458cce 79957b372307 00f454bbac01 2024-02-07T12:47:31.397 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (28m) 103s ago 28m 61.4M 2048M 17.2.0 e1d6a67b021e b9825d0db32d 2024-02-07T12:47:31.397 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (2m) 103s ago 24m 8632k - 1.5.0 0da6a335fe13 c4bdcb1c4d49 2024-02-07T12:47:31.397 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (2m) 21s ago 24m 16.3M - 1.5.0 0da6a335fe13 04ea81f79d85 2024-02-07T12:47:31.397 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (27m) 103s ago 27m 54.4M 2993M 17.2.0 e1d6a67b021e 87f2560a9b15 2024-02-07T12:47:31.397 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (27m) 103s ago 27m 57.0M 2993M 17.2.0 e1d6a67b021e 0f9a190f0e0d 2024-02-07T12:47:31.397 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (27m) 103s ago 27m 52.7M 2993M 17.2.0 e1d6a67b021e 7db94f25a1aa 2024-02-07T12:47:31.397 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (26m) 103s ago 26m 53.2M 2993M 17.2.0 e1d6a67b021e 0b6b674188ed 2024-02-07T12:47:31.397 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (26m) 21s ago 26m 55.5M 3248M 17.2.0 e1d6a67b021e df72cfd69b3b 2024-02-07T12:47:31.397 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (25m) 21s ago 25m 52.4M 3248M 17.2.0 e1d6a67b021e 3bdb22c07537 2024-02-07T12:47:31.397 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (25m) 21s ago 25m 52.0M 3248M 17.2.0 e1d6a67b021e 7859b651e0d9 2024-02-07T12:47:31.398 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (25m) 21s ago 25m 55.2M 3248M 17.2.0 e1d6a67b021e b301e7c75d73 2024-02-07T12:47:31.398 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (2m) 21s ago 24m 56.7M - 2.43.0 a07b618ecd1d c9b607eea1be 2024-02-07T12:47:31.398 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (22m) 103s ago 22m 77.8M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:47:31.398 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (22m) 21s ago 22m 78.0M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:47:31.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:31 smithi019 ceph-mon[29267]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:47:31.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:31 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:47:31.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:31 smithi019 ceph-mon[24574]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:47:31.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:31 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:47:31.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:31 smithi172 ceph-mon[67636]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:47:31.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:31 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:47:31.944 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 2'"'"'' 2024-02-07T12:47:33.514 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:33 smithi019 ceph-mon[24574]: from='client.25471 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:47:33.514 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:33 smithi019 ceph-mon[24574]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:47:33.514 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:33 smithi019 ceph-mon[29267]: from='client.25471 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:47:33.515 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:33 smithi019 ceph-mon[29267]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:47:33.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:33 smithi172 ceph-mon[67636]: from='client.25471 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:47:33.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:33 smithi172 ceph-mon[67636]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:47:33.927 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:47:34.192 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:47:33] "GET /metrics HTTP/1.1" 200 38002 "" "Prometheus/2.43.0" 2024-02-07T12:47:34.484 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:34 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/897747572' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:34.485 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:34 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/897747572' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:34.510 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch upgrade status' 2024-02-07T12:47:34.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:34 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/897747572' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:35.483 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:35 smithi019 ceph-mon[29267]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:47:35.484 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:35 smithi019 ceph-mon[24574]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:47:35.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:35 smithi172 ceph-mon[67636]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:47:36.564 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:47:36.564 INFO:teuthology.orchestra.run.smithi019.stdout: "target_image": null, 2024-02-07T12:47:36.564 INFO:teuthology.orchestra.run.smithi019.stdout: "in_progress": false, 2024-02-07T12:47:36.564 INFO:teuthology.orchestra.run.smithi019.stdout: "which": "", 2024-02-07T12:47:36.564 INFO:teuthology.orchestra.run.smithi019.stdout: "services_complete": [], 2024-02-07T12:47:36.564 INFO:teuthology.orchestra.run.smithi019.stdout: "progress": null, 2024-02-07T12:47:36.564 INFO:teuthology.orchestra.run.smithi019.stdout: "message": "", 2024-02-07T12:47:36.564 INFO:teuthology.orchestra.run.smithi019.stdout: "is_paused": false 2024-02-07T12:47:36.564 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:47:37.234 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph health detail' 2024-02-07T12:47:37.548 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:37 smithi019 ceph-mon[29267]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:47:37.548 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:37 smithi019 ceph-mon[24574]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:47:37.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:37 smithi172 ceph-mon[67636]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:47:38.500 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:38 smithi019 ceph-mon[24574]: from='client.15597 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:47:38.501 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:38 smithi019 ceph-mon[29267]: from='client.15597 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:47:38.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:38 smithi172 ceph-mon[67636]: from='client.15597 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:47:39.328 INFO:teuthology.orchestra.run.smithi019.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-02-07T12:47:39.328 INFO:teuthology.orchestra.run.smithi019.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-02-07T12:47:39.328 INFO:teuthology.orchestra.run.smithi019.stdout: daemon iscsi.foo.smithi019.svjimj on smithi019 is in unknown state 2024-02-07T12:47:39.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:39 smithi019 ceph-mon[24574]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:47:39.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:39 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:47:39.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:39 smithi019 ceph-mon[29267]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:47:39.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:39 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:47:39.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:39 smithi172 ceph-mon[67636]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:47:39.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:39 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:47:39.912 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '"'"'{print $2}'"'"')' 2024-02-07T12:47:40.500 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:40 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/1157204144' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:47:40.501 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:40 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/1157204144' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:47:40.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:40 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/1157204144' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:47:41.384 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:41 smithi019 ceph-mon[29267]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:47:41.384 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:41 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:47:41.384 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:41 smithi019 ceph-mon[24574]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:47:41.384 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:41 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:47:41.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:41 smithi172 ceph-mon[67636]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:47:41.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:41 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:47:43.438 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:43 smithi019 ceph-mon[24574]: from='client.25495 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:47:43.438 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:43 smithi019 ceph-mon[24574]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:47:43.438 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:43 smithi019 ceph-mon[24574]: from='client.25501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "daemon_types": "mon", "hosts": "smithi019", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:47:43.438 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:43 smithi019 ceph-mon[29267]: from='client.25495 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:47:43.439 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:43 smithi019 ceph-mon[29267]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:47:43.439 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:43 smithi019 ceph-mon[29267]: from='client.25501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "daemon_types": "mon", "hosts": "smithi019", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:47:43.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:43 smithi172 ceph-mon[67636]: from='client.25495 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:47:43.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:43 smithi172 ceph-mon[67636]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:47:43.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:43 smithi172 ceph-mon[67636]: from='client.25501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "daemon_types": "mon", "hosts": "smithi019", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:47:44.293 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:47:43] "GET /metrics HTTP/1.1" 200 38005 "" "Prometheus/2.43.0" 2024-02-07T12:47:44.374 INFO:teuthology.orchestra.run.smithi019.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:47:44.935 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-02-07T12:47:45.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:45 smithi019 ceph-mon[24574]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:47:45.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:45 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:45.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:45 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:47:45.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:45 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:45.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:45 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:47:45.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:45 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:45.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:45 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:47:45.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:45 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:47:45.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:45 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:45.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:45 smithi019 ceph-mon[29267]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:47:45.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:45 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:45.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:45 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:47:45.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:45 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:45.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:45 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:47:45.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:45 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:45.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:45 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:47:45.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:45 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:47:45.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:45 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:45.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:45 smithi172 ceph-mon[67636]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:47:45.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:45 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:45.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:45 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:47:45.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:45 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:45.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:45 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:47:45.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:45 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:45.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:45 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:47:45.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:45 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:47:45.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:45 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:46.577 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:46 smithi172 ceph-mon[67636]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:47:46.577 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:46 smithi172 ceph-mon[67636]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 985 B/s rd, 0 op/s 2024-02-07T12:47:46.577 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:46 smithi172 ceph-mon[67636]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 610 B/s rd, 0 op/s 2024-02-07T12:47:46.577 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:46 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:47:46.577 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:46 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:47:46.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:46 smithi019 ceph-mon[24574]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:47:46.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:46 smithi019 ceph-mon[24574]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 985 B/s rd, 0 op/s 2024-02-07T12:47:46.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:46 smithi019 ceph-mon[24574]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 610 B/s rd, 0 op/s 2024-02-07T12:47:46.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:46 smithi019 ceph-mon[24574]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:47:46.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:46 smithi019 ceph-mon[24574]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:47:46.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:46 smithi019 ceph-mon[29267]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:47:46.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:46 smithi019 ceph-mon[29267]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 985 B/s rd, 0 op/s 2024-02-07T12:47:46.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:46 smithi019 ceph-mon[29267]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 610 B/s rd, 0 op/s 2024-02-07T12:47:46.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:46 smithi019 ceph-mon[29267]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:47:46.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:46 smithi019 ceph-mon[29267]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:47:47.300 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:47:47.125+0000 7eff60d5c700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:47.300 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:47:47.300 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:47.301 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:47.301 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:47.301 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:47.301 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:47.301 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:47:47.301 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:47.301 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:47:47.301 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:47.301 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:47:47.301 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:47.301 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:47.301 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:47.302 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:47.302 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:47.302 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:47.302 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:47.302 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-mon[29267]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:47:47.302 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-mon[29267]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:47.302 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:47:47.303 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:47.303 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:47.303 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:47.303 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:47.303 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:47.303 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:47:47.303 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:47.303 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:47:47.303 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:47.303 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:47:47.303 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:47.303 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:47.303 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:47.304 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:47.304 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:47.304 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:47.304 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:47.304 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:47.304 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:47.304 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:47:47.304 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:47:47.304 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:47:47.304 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:47:47.304 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:47:47.304 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:47:47.304 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:47:47.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:47:47.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:47:47.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:47:47.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:47:47.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:47:47.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:47:47.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:47:47.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:47:47.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:47:47.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:47:47.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:47.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:47:47.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:47.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:47.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:47.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:47.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:47.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:47:47.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:47.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:47:47.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:47.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:47:47.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:47.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:47.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:47.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:47.307 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:47.307 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:47.307 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:47.307 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:47.307 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:47.307 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-mon[29267]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:47:47.307 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:47.307 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:47.307 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:47:47.308 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:47:47.308 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:47:47.308 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:47:47.308 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:47:47.308 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:47:47.308 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:47:47.308 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:47:47.308 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:47:47.308 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:47:47.308 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:47:47.308 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:47:47.308 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:47:47.308 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:47:47.308 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:47:47.308 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:47:47.309 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:47:47.309 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:47.309 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:47:47.309 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:47.309 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:47.309 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:47.309 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:47.309 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:47.309 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:47:47.309 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:47.309 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:47:47.309 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:47.309 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:47:47.309 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:47.310 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:47.310 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:47.310 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:47.310 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:47.310 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:47.310 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:47.310 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:47.310 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:47.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-mon[24574]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:47:47.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-mon[24574]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:47.594 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:47:47.594 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:47.594 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:47.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:47.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:47.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:47.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:47:47.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:47.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:47:47.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:47.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:47:47.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:47.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:47.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:47.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:47.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:47.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:47.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:47.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:47.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:47.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:47:47.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:47:47.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:47:47.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:47:47.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:47:47.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:47:47.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:47:47.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:47:47.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:47:47.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:47:47.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:47:47.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:47:47.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:47:47.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:47:47.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:47:47.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:47:47.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:47:47.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:47.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:47:47.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:47.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:47.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:47.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:47.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:47.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:47:47.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:47.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:47:47.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:47.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:47:47.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:47.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:47.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:47.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:47.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:47.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:47.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:47.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:47.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:47.598 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:47 smithi019 ceph-mon[24574]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:47:47.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:47 smithi172 ceph-mon[67636]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:47:47.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:47 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:47.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:47:47.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:47.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:47.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:47.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:47.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:47.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:47:47.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:47.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:47:47.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:47.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:47:47.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:47.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:47.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:47.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:47.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:47.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:47.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:47.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:47.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:47.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:47:47.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:47:47.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:47:47.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:47:47.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:47:47.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:47:47.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:47:47.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:47:47.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:47:47.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:47:47.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:47:47.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:47:47.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:47:47.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:47:47.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:47:47.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:47:47.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:47:47.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:47:47.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:47:47.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:47:47.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:47:47.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:47:47.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:47.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:47.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:47:47.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:47:47.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:47:47.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:47:47.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:47:47.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:47:47.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:47:47.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:47:47.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:47:47.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:47:47.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:47:47.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:47:47.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:47:47.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:47:47.623 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:47 smithi172 ceph-mon[67636]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:47:48.221 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:47:48.924 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:47:48.924 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (2m) 2m ago 23m 17.3M - 0.25.0 c8568f914cd2 fec9255f35e7 2024-02-07T12:47:48.924 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (108s) 38s ago 23m 80.1M - 9.4.12 a0ea5e3c46d9 aaffac63bc28 2024-02-07T12:47:48.924 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 unknown 2m ago 23m 57.2M - 3.5 e1d6a67b021e 5113aa77e3f2 2024-02-07T12:47:48.924 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283,8765 running (3m) 38s ago 28m 452M - 19.0.0-1252-g44458cce 79957b372307 5aa0c04fdfd7 2024-02-07T12:47:48.924 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (13m) 2m ago 30m 503M - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:47:48.924 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (30m) 2m ago 30m 79.6M 2048M 17.2.0 e1d6a67b021e f3a8a050cf1d 2024-02-07T12:47:48.924 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (43s) 38s ago 28m 20.4M 2048M 19.0.0-1252-g44458cce 79957b372307 00f454bbac01 2024-02-07T12:47:48.924 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (28m) 2m ago 28m 61.4M 2048M 17.2.0 e1d6a67b021e b9825d0db32d 2024-02-07T12:47:48.924 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (2m) 2m ago 24m 8632k - 1.5.0 0da6a335fe13 c4bdcb1c4d49 2024-02-07T12:47:48.924 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (2m) 38s ago 24m 16.3M - 1.5.0 0da6a335fe13 04ea81f79d85 2024-02-07T12:47:48.924 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (28m) 2m ago 28m 54.4M 2993M 17.2.0 e1d6a67b021e 87f2560a9b15 2024-02-07T12:47:48.925 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (27m) 2m ago 27m 57.0M 2993M 17.2.0 e1d6a67b021e 0f9a190f0e0d 2024-02-07T12:47:48.925 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (27m) 2m ago 27m 52.7M 2993M 17.2.0 e1d6a67b021e 7db94f25a1aa 2024-02-07T12:47:48.925 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (26m) 2m ago 26m 53.2M 2993M 17.2.0 e1d6a67b021e 0b6b674188ed 2024-02-07T12:47:48.925 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (26m) 38s ago 26m 55.5M 3248M 17.2.0 e1d6a67b021e df72cfd69b3b 2024-02-07T12:47:48.925 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (26m) 38s ago 26m 52.4M 3248M 17.2.0 e1d6a67b021e 3bdb22c07537 2024-02-07T12:47:48.925 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (25m) 38s ago 25m 52.0M 3248M 17.2.0 e1d6a67b021e 7859b651e0d9 2024-02-07T12:47:48.925 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (25m) 38s ago 25m 55.2M 3248M 17.2.0 e1d6a67b021e b301e7c75d73 2024-02-07T12:47:48.925 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (2m) 38s ago 24m 56.7M - 2.43.0 a07b618ecd1d c9b607eea1be 2024-02-07T12:47:48.925 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (23m) 2m ago 23m 77.8M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:47:48.925 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (23m) 38s ago 23m 78.0M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:47:49.336 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:47:49.336 INFO:teuthology.orchestra.run.smithi019.stdout: "mon": { 2024-02-07T12:47:49.336 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-02-07T12:47:49.336 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 1 2024-02-07T12:47:49.336 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:47:49.337 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr": { 2024-02-07T12:47:49.337 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:47:49.337 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:47:49.337 INFO:teuthology.orchestra.run.smithi019.stdout: "osd": { 2024-02-07T12:47:49.337 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-02-07T12:47:49.337 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:47:49.337 INFO:teuthology.orchestra.run.smithi019.stdout: "mds": {}, 2024-02-07T12:47:49.337 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw": { 2024-02-07T12:47:49.337 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:47:49.337 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:47:49.337 INFO:teuthology.orchestra.run.smithi019.stdout: "overall": { 2024-02-07T12:47:49.337 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-02-07T12:47:49.337 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 3 2024-02-07T12:47:49.337 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:47:49.337 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:47:49.338 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:49 smithi019 ceph-mon[24574]: from='client.15621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:47:49.338 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:49 smithi019 ceph-mon[29267]: from='client.15621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:47:49.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:49 smithi172 ceph-mon[67636]: from='client.15621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:47:49.680 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:47:49.680 INFO:teuthology.orchestra.run.smithi019.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", 2024-02-07T12:47:49.681 INFO:teuthology.orchestra.run.smithi019.stdout: "in_progress": true, 2024-02-07T12:47:49.681 INFO:teuthology.orchestra.run.smithi019.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi019", 2024-02-07T12:47:49.681 INFO:teuthology.orchestra.run.smithi019.stdout: "services_complete": [], 2024-02-07T12:47:49.681 INFO:teuthology.orchestra.run.smithi019.stdout: "progress": "", 2024-02-07T12:47:49.681 INFO:teuthology.orchestra.run.smithi019.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 image", 2024-02-07T12:47:49.681 INFO:teuthology.orchestra.run.smithi019.stdout: "is_paused": false 2024-02-07T12:47:49.681 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:47:50.452 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:50 smithi019 ceph-mon[24574]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 610 B/s rd, 0 op/s 2024-02-07T12:47:50.452 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:50 smithi019 ceph-mon[24574]: from='client.25513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:47:50.452 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:50 smithi019 ceph-mon[24574]: from='client.15633 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:47:50.452 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:50 smithi019 ceph-mon[24574]: from='client.? 172.21.15.19:0/810666013' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:50.452 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:50 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:50.452 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:50 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:47:50.452 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:50 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:50.452 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:50 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:50.452 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:50 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-02-07T12:47:50.453 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:50 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-02-07T12:47:50.453 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:50 smithi019 ceph-mon[29267]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 610 B/s rd, 0 op/s 2024-02-07T12:47:50.453 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:50 smithi019 ceph-mon[29267]: from='client.25513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:47:50.453 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:50 smithi019 ceph-mon[29267]: from='client.15633 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:47:50.453 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:50 smithi019 ceph-mon[29267]: from='client.? 172.21.15.19:0/810666013' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:50.453 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:50 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:50.453 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:50 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:47:50.453 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:50 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:50.453 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:50 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:50.453 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:50 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-02-07T12:47:50.453 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:50 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-02-07T12:47:50.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:50 smithi172 ceph-mon[67636]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 610 B/s rd, 0 op/s 2024-02-07T12:47:50.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:50 smithi172 ceph-mon[67636]: from='client.25513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:47:50.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:50 smithi172 ceph-mon[67636]: from='client.15633 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:47:50.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:50 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/810666013' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:50.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:50 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:50.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:50 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:47:50.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:50 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:47:50.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:50 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:50.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:50 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-02-07T12:47:50.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:50 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-02-07T12:47:51.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:51 smithi019 ceph-mon[24574]: from='client.15645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:47:51.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:51 smithi019 ceph-mon[24574]: Upgrade: Target is version 19.0.0-1252-g44458cce (squid) 2024-02-07T12:47:51.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:51 smithi019 ceph-mon[24574]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe'] 2024-02-07T12:47:51.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:51 smithi019 ceph-mon[24574]: Upgrade: Setting container_image for all mgr 2024-02-07T12:47:51.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:51 smithi019 ceph-mon[24574]: Upgrade: It appears safe to stop mon.a 2024-02-07T12:47:51.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:51 smithi019 ceph-mon[24574]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:47:51.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:51 smithi019 ceph-mon[29267]: from='client.15645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:47:51.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:51 smithi019 ceph-mon[29267]: Upgrade: Target is version 19.0.0-1252-g44458cce (squid) 2024-02-07T12:47:51.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:51 smithi019 ceph-mon[29267]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe'] 2024-02-07T12:47:51.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:51 smithi019 ceph-mon[29267]: Upgrade: Setting container_image for all mgr 2024-02-07T12:47:51.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:51 smithi019 ceph-mon[29267]: Upgrade: It appears safe to stop mon.a 2024-02-07T12:47:51.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:51 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:47:51.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:51 smithi172 ceph-mon[67636]: from='client.15645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:47:51.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:51 smithi172 ceph-mon[67636]: Upgrade: Target is version 19.0.0-1252-g44458cce (squid) 2024-02-07T12:47:51.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:51 smithi172 ceph-mon[67636]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe'] 2024-02-07T12:47:51.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:51 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all mgr 2024-02-07T12:47:51.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:51 smithi172 ceph-mon[67636]: Upgrade: It appears safe to stop mon.a 2024-02-07T12:47:51.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:51 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:47:52.590 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:52 smithi019 ceph-mon[24574]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:47:52.591 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:52 smithi019 ceph-mon[24574]: Upgrade: Updating mon.a 2024-02-07T12:47:52.591 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:52 smithi019 ceph-mon[24574]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:52.591 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:52 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:47:52.591 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:52 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:47:52.591 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:52 smithi019 ceph-mon[24574]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:52.591 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:52 smithi019 ceph-mon[29267]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:47:52.591 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:52 smithi019 ceph-mon[29267]: Upgrade: Updating mon.a 2024-02-07T12:47:52.591 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:52 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:52.591 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:52 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:47:52.591 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:52 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:47:52.591 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:52 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:52.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:52 smithi172 ceph-mon[67636]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:47:52.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:52 smithi172 ceph-mon[67636]: Upgrade: Updating mon.a 2024-02-07T12:47:52.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:52 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='mgr.y' 2024-02-07T12:47:52.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:52 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:47:52.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:52 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:47:52.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:52 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:47:53.388 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:53 smithi019 systemd[1]: Stopping Ceph mon.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:47:53.388 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:53 smithi019 ceph-mon[24574]: Deploying daemon mon.a on smithi019 2024-02-07T12:47:53.388 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:53 smithi019 ceph-mon[29267]: Deploying daemon mon.a on smithi019 2024-02-07T12:47:53.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:53 smithi172 ceph-mon[67636]: Deploying daemon mon.a on smithi019 2024-02-07T12:47:53.670 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-a[24570]: 2024-02-07T12:47:53.384+0000 7fbc7d653700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:47:53.670 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-a[24570]: 2024-02-07T12:47:53.384+0000 7fbc7d653700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-02-07T12:47:53.670 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:53 smithi019 podman[82756]: 2024-02-07 12:47:53.405479842 +0000 UTC m=+0.144737825 container died f3a8a050cf1d560b64ce22cb4e363436d142dcdb71fe1e93d919b7adc4d6fcd8 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-a, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, vcs-type=git, vendor=Red Hat, Inc., architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, name=centos-stream, GIT_CLEAN=True, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux ) 2024-02-07T12:47:54.036 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:47:53] "GET /metrics HTTP/1.1" 200 38005 "" "Prometheus/2.43.0" 2024-02-07T12:47:54.037 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:53 smithi019 podman[82756]: 2024-02-07 12:47:53.767675088 +0000 UTC m=+0.506933079 container cleanup f3a8a050cf1d560b64ce22cb4e363436d142dcdb71fe1e93d919b7adc4d6fcd8 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, io.openshift.expose-services=, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, distribution-scope=public, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, GIT_BRANCH=HEAD, vcs-type=git, io.openshift.tags=base centos centos-stream) 2024-02-07T12:47:54.037 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:53 smithi019 bash[82756]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-a 2024-02-07T12:47:54.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 podman[82770]: 2024-02-07 12:47:54.035489554 +0000 UTC m=+0.624827052 container remove f3a8a050cf1d560b64ce22cb4e363436d142dcdb71fe1e93d919b7adc4d6fcd8 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-a, ceph=True, GIT_CLEAN=True, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, name=centos-stream, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, io.buildah.version=1.19.8, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, version=8, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0) 2024-02-07T12:47:54.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mon.a.service: Deactivated successfully. 2024-02-07T12:47:54.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 systemd[1]: Stopped Ceph mon.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:47:54.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mon.a.service: Consumed 25.676s CPU time. 2024-02-07T12:47:54.696 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 systemd[1]: Starting Ceph mon.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:47:54.696 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 podman[82899]: 2024-02-07 12:47:54.603865219 +0000 UTC m=+0.110885219 container create 8b1b0e21b1c548070914ac24243372f162013338f674502ce47637e51c54119d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-a, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.2) 2024-02-07T12:47:54.696 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 podman[82899]: 2024-02-07 12:47:54.511219967 +0000 UTC m=+0.018239933 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:47:55.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 podman[82899]: 2024-02-07 12:47:54.781383211 +0000 UTC m=+0.288403168 container init 8b1b0e21b1c548070914ac24243372f162013338f674502ce47637e51c54119d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-a, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , io.buildah.version=1.33.2, org.label-schema.build-date=20240131, ceph=True, GIT_BRANCH=HEAD) 2024-02-07T12:47:55.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 podman[82899]: 2024-02-07 12:47:54.784341551 +0000 UTC m=+0.291361511 container start 8b1b0e21b1c548070914ac24243372f162013338f674502ce47637e51c54119d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-a, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-02-07T12:47:55.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: set uid:gid to 167:167 (ceph:ceph) 2024-02-07T12:47:55.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev), process ceph-mon, pid 2 2024-02-07T12:47:55.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: pidfile_write: ignore empty --pid-file 2024-02-07T12:47:55.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: load: jerasure load: lrc 2024-02-07T12:47:55.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: RocksDB version: 7.9.2 2024-02-07T12:47:55.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Git sha 0 2024-02-07T12:47:55.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Compile date 2024-02-05 19:41:51 2024-02-07T12:47:55.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: DB SUMMARY 2024-02-07T12:47:55.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: DB Session ID: IY0CMSS65G7P3S9LXC4H 2024-02-07T12:47:55.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: CURRENT file: CURRENT 2024-02-07T12:47:55.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: IDENTITY file: IDENTITY 2024-02-07T12:47:55.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: MANIFEST file: MANIFEST-000015 size: 2889 Bytes 2024-02-07T12:47:55.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000060.sst 2024-02-07T12:47:55.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000058.log size: 2466295 ; 2024-02-07T12:47:55.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.error_if_exists: 0 2024-02-07T12:47:55.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.create_if_missing: 0 2024-02-07T12:47:55.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.paranoid_checks: 1 2024-02-07T12:47:55.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.flush_verify_memtable_count: 1 2024-02-07T12:47:55.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-02-07T12:47:55.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-02-07T12:47:55.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.env: 0x5565c82d0ae0 2024-02-07T12:47:55.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.fs: PosixFileSystem 2024-02-07T12:47:55.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.info_log: 0x5565ca2c38c0 2024-02-07T12:47:55.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.max_file_opening_threads: 16 2024-02-07T12:47:55.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.statistics: (nil) 2024-02-07T12:47:55.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.use_fsync: 0 2024-02-07T12:47:55.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.max_log_file_size: 0 2024-02-07T12:47:55.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-02-07T12:47:55.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.log_file_time_to_roll: 0 2024-02-07T12:47:55.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.keep_log_file_num: 1000 2024-02-07T12:47:55.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.recycle_log_file_num: 0 2024-02-07T12:47:55.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.allow_fallocate: 1 2024-02-07T12:47:55.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.allow_mmap_reads: 0 2024-02-07T12:47:55.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.allow_mmap_writes: 0 2024-02-07T12:47:55.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.use_direct_reads: 0 2024-02-07T12:47:55.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-02-07T12:47:55.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.create_missing_column_families: 0 2024-02-07T12:47:55.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.db_log_dir: 2024-02-07T12:47:55.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.wal_dir: 2024-02-07T12:47:55.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.table_cache_numshardbits: 6 2024-02-07T12:47:55.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.WAL_ttl_seconds: 0 2024-02-07T12:47:55.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.WAL_size_limit_MB: 0 2024-02-07T12:47:55.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-02-07T12:47:55.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-02-07T12:47:55.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.is_fd_close_on_exec: 1 2024-02-07T12:47:55.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.advise_random_on_open: 1 2024-02-07T12:47:55.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.db_write_buffer_size: 0 2024-02-07T12:47:55.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.write_buffer_manager: 0x5565ca296dc0 2024-02-07T12:47:55.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-02-07T12:47:55.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-02-07T12:47:55.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.use_adaptive_mutex: 0 2024-02-07T12:47:55.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.rate_limiter: (nil) 2024-02-07T12:47:55.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-02-07T12:47:55.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.wal_recovery_mode: 2 2024-02-07T12:47:55.099 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.enable_thread_tracking: 0 2024-02-07T12:47:55.099 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.enable_pipelined_write: 0 2024-02-07T12:47:55.099 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.unordered_write: 0 2024-02-07T12:47:55.099 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-02-07T12:47:55.099 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-02-07T12:47:55.099 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-02-07T12:47:55.099 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-02-07T12:47:55.099 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.row_cache: None 2024-02-07T12:47:55.099 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.wal_filter: None 2024-02-07T12:47:55.100 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-02-07T12:47:55.100 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.allow_ingest_behind: 0 2024-02-07T12:47:55.100 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.two_write_queues: 0 2024-02-07T12:47:55.100 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.manual_wal_flush: 0 2024-02-07T12:47:55.101 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.wal_compression: 0 2024-02-07T12:47:55.101 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.atomic_flush: 0 2024-02-07T12:47:55.101 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-02-07T12:47:55.101 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.persist_stats_to_disk: 0 2024-02-07T12:47:55.101 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.write_dbid_to_manifest: 0 2024-02-07T12:47:55.101 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.log_readahead_size: 0 2024-02-07T12:47:55.101 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-02-07T12:47:55.101 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.best_efforts_recovery: 0 2024-02-07T12:47:55.101 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-02-07T12:47:55.101 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-02-07T12:47:55.101 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.allow_data_in_errors: 0 2024-02-07T12:47:55.101 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.db_host_id: __hostname__ 2024-02-07T12:47:55.101 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.enforce_single_del_contracts: true 2024-02-07T12:47:55.101 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.max_background_jobs: 2 2024-02-07T12:47:55.101 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.max_background_compactions: -1 2024-02-07T12:47:55.102 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.max_subcompactions: 1 2024-02-07T12:47:55.102 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-02-07T12:47:55.102 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-02-07T12:47:55.102 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.delayed_write_rate : 16777216 2024-02-07T12:47:55.102 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.max_total_wal_size: 0 2024-02-07T12:47:55.102 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-02-07T12:47:55.102 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.stats_dump_period_sec: 600 2024-02-07T12:47:55.102 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.stats_persist_period_sec: 600 2024-02-07T12:47:55.102 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-02-07T12:47:55.102 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.max_open_files: -1 2024-02-07T12:47:55.102 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.bytes_per_sync: 0 2024-02-07T12:47:55.102 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.wal_bytes_per_sync: 0 2024-02-07T12:47:55.102 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.strict_bytes_per_sync: 0 2024-02-07T12:47:55.102 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.compaction_readahead_size: 0 2024-02-07T12:47:55.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.max_background_flushes: -1 2024-02-07T12:47:55.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Compression algorithms supported: 2024-02-07T12:47:55.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-02-07T12:47:55.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: kZSTD supported: 0 2024-02-07T12:47:55.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: kXpressCompression supported: 0 2024-02-07T12:47:55.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: kLZ4HCCompression supported: 1 2024-02-07T12:47:55.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: kZlibCompression supported: 1 2024-02-07T12:47:55.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: kSnappyCompression supported: 1 2024-02-07T12:47:55.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: kLZ4Compression supported: 1 2024-02-07T12:47:55.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: kBZip2Compression supported: 0 2024-02-07T12:47:55.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Fast CRC32 supported: Supported on x86 2024-02-07T12:47:55.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: DMutex implementation: pthread_mutex_t 2024-02-07T12:47:55.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-02-07T12:47:55.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-02-07T12:47:55.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-02-07T12:47:55.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.merge_operator: 2024-02-07T12:47:55.104 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.compaction_filter: None 2024-02-07T12:47:55.104 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.compaction_filter_factory: None 2024-02-07T12:47:55.104 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.sst_partitioner_factory: None 2024-02-07T12:47:55.104 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.memtable_factory: SkipListFactory 2024-02-07T12:47:55.104 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.table_factory: BlockBasedTable 2024-02-07T12:47:55.104 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5565ca2c39c0) 2024-02-07T12:47:55.104 INFO:journalctl@ceph.mon.a.smithi019.stdout: cache_index_and_filter_blocks: 1 2024-02-07T12:47:55.104 INFO:journalctl@ceph.mon.a.smithi019.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-02-07T12:47:55.104 INFO:journalctl@ceph.mon.a.smithi019.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-02-07T12:47:55.104 INFO:journalctl@ceph.mon.a.smithi019.stdout: pin_top_level_index_and_filter: 1 2024-02-07T12:47:55.104 INFO:journalctl@ceph.mon.a.smithi019.stdout: index_type: 0 2024-02-07T12:47:55.104 INFO:journalctl@ceph.mon.a.smithi019.stdout: data_block_index_type: 0 2024-02-07T12:47:55.104 INFO:journalctl@ceph.mon.a.smithi019.stdout: index_shortening: 1 2024-02-07T12:47:55.104 INFO:journalctl@ceph.mon.a.smithi019.stdout: data_block_hash_table_util_ratio: 0.750000 2024-02-07T12:47:55.105 INFO:journalctl@ceph.mon.a.smithi019.stdout: checksum: 4 2024-02-07T12:47:55.105 INFO:journalctl@ceph.mon.a.smithi019.stdout: no_block_cache: 0 2024-02-07T12:47:55.105 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_cache: 0x5565ca2c7090 2024-02-07T12:47:55.105 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_cache_name: BinnedLRUCache 2024-02-07T12:47:55.105 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_cache_options: 2024-02-07T12:47:55.105 INFO:journalctl@ceph.mon.a.smithi019.stdout: capacity : 536870912 2024-02-07T12:47:55.105 INFO:journalctl@ceph.mon.a.smithi019.stdout: num_shard_bits : 4 2024-02-07T12:47:55.105 INFO:journalctl@ceph.mon.a.smithi019.stdout: strict_capacity_limit : 0 2024-02-07T12:47:55.105 INFO:journalctl@ceph.mon.a.smithi019.stdout: high_pri_pool_ratio: 0.000 2024-02-07T12:47:55.105 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_cache_compressed: (nil) 2024-02-07T12:47:55.105 INFO:journalctl@ceph.mon.a.smithi019.stdout: persistent_cache: (nil) 2024-02-07T12:47:55.105 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_size: 4096 2024-02-07T12:47:55.105 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_size_deviation: 10 2024-02-07T12:47:55.105 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_restart_interval: 16 2024-02-07T12:47:55.106 INFO:journalctl@ceph.mon.a.smithi019.stdout: index_block_restart_interval: 1 2024-02-07T12:47:55.106 INFO:journalctl@ceph.mon.a.smithi019.stdout: metadata_block_size: 4096 2024-02-07T12:47:55.106 INFO:journalctl@ceph.mon.a.smithi019.stdout: partition_filters: 0 2024-02-07T12:47:55.106 INFO:journalctl@ceph.mon.a.smithi019.stdout: use_delta_encoding: 1 2024-02-07T12:47:55.106 INFO:journalctl@ceph.mon.a.smithi019.stdout: filter_policy: bloomfilter 2024-02-07T12:47:55.106 INFO:journalctl@ceph.mon.a.smithi019.stdout: whole_key_filtering: 1 2024-02-07T12:47:55.106 INFO:journalctl@ceph.mon.a.smithi019.stdout: verify_compression: 0 2024-02-07T12:47:55.106 INFO:journalctl@ceph.mon.a.smithi019.stdout: read_amp_bytes_per_bit: 0 2024-02-07T12:47:55.106 INFO:journalctl@ceph.mon.a.smithi019.stdout: format_version: 5 2024-02-07T12:47:55.106 INFO:journalctl@ceph.mon.a.smithi019.stdout: enable_index_compression: 1 2024-02-07T12:47:55.106 INFO:journalctl@ceph.mon.a.smithi019.stdout: block_align: 0 2024-02-07T12:47:55.106 INFO:journalctl@ceph.mon.a.smithi019.stdout: max_auto_readahead_size: 262144 2024-02-07T12:47:55.106 INFO:journalctl@ceph.mon.a.smithi019.stdout: prepopulate_block_cache: 0 2024-02-07T12:47:55.106 INFO:journalctl@ceph.mon.a.smithi019.stdout: initial_auto_readahead_size: 8192 2024-02-07T12:47:55.107 INFO:journalctl@ceph.mon.a.smithi019.stdout: num_file_reads_for_auto_readahead: 2 2024-02-07T12:47:55.107 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.write_buffer_size: 33554432 2024-02-07T12:47:55.107 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.max_write_buffer_number: 2 2024-02-07T12:47:55.107 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.compression: NoCompression 2024-02-07T12:47:55.107 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.bottommost_compression: Disabled 2024-02-07T12:47:55.107 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.prefix_extractor: nullptr 2024-02-07T12:47:55.107 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-02-07T12:47:55.107 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.num_levels: 7 2024-02-07T12:47:55.107 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-02-07T12:47:55.107 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-02-07T12:47:55.107 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-02-07T12:47:55.107 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-02-07T12:47:55.107 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-02-07T12:47:55.107 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-02-07T12:47:55.108 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-02-07T12:47:55.108 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-02-07T12:47:55.108 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-02-07T12:47:55.108 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-02-07T12:47:55.108 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-02-07T12:47:55.108 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-02-07T12:47:55.108 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.compression_opts.window_bits: -14 2024-02-07T12:47:55.108 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.compression_opts.level: 32767 2024-02-07T12:47:55.108 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.compression_opts.strategy: 0 2024-02-07T12:47:55.108 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-02-07T12:47:55.108 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-02-07T12:47:55.108 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-02-07T12:47:55.108 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-02-07T12:47:55.108 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.compression_opts.enabled: false 2024-02-07T12:47:55.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-02-07T12:47:55.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-02-07T12:47:55.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-02-07T12:47:55.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-02-07T12:47:55.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.target_file_size_base: 67108864 2024-02-07T12:47:55.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.target_file_size_multiplier: 1 2024-02-07T12:47:55.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-02-07T12:47:55.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-02-07T12:47:55.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-02-07T12:47:55.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-02-07T12:47:55.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-02-07T12:47:55.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-02-07T12:47:55.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-02-07T12:47:55.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-02-07T12:47:55.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-02-07T12:47:55.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-02-07T12:47:55.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-02-07T12:47:55.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-02-07T12:47:55.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-02-07T12:47:55.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.arena_block_size: 1048576 2024-02-07T12:47:55.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-02-07T12:47:55.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-02-07T12:47:55.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.disable_auto_compactions: 0 2024-02-07T12:47:55.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-02-07T12:47:55.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-02-07T12:47:55.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-02-07T12:47:55.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-02-07T12:47:55.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-02-07T12:47:55.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-02-07T12:47:55.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-02-07T12:47:55.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-02-07T12:47:55.111 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-02-07T12:47:55.111 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-02-07T12:47:55.111 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.table_properties_collectors: 2024-02-07T12:47:55.111 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.inplace_update_support: 0 2024-02-07T12:47:55.111 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.inplace_update_num_locks: 10000 2024-02-07T12:47:55.111 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-02-07T12:47:55.111 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-02-07T12:47:55.111 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.memtable_huge_page_size: 0 2024-02-07T12:47:55.111 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.bloom_locality: 0 2024-02-07T12:47:55.111 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.max_successive_merges: 0 2024-02-07T12:47:55.111 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.optimize_filters_for_hits: 0 2024-02-07T12:47:55.111 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.paranoid_file_checks: 0 2024-02-07T12:47:55.111 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.force_consistency_checks: 1 2024-02-07T12:47:55.111 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.report_bg_io_stats: 0 2024-02-07T12:47:55.111 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.ttl: 2592000 2024-02-07T12:47:55.112 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.periodic_compaction_seconds: 0 2024-02-07T12:47:55.112 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-02-07T12:47:55.112 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-02-07T12:47:55.112 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.enable_blob_files: false 2024-02-07T12:47:55.112 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.min_blob_size: 0 2024-02-07T12:47:55.112 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.blob_file_size: 268435456 2024-02-07T12:47:55.112 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.blob_compression_type: NoCompression 2024-02-07T12:47:55.112 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.enable_blob_garbage_collection: false 2024-02-07T12:47:55.112 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-02-07T12:47:55.112 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-02-07T12:47:55.112 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-02-07T12:47:55.112 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.blob_file_starting_level: 0 2024-02-07T12:47:55.112 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-02-07T12:47:55.112 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 60.sst 2024-02-07T12:47:55.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-02-07T12:47:55.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 62, last_sequence is 26788, log_number is 58,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-02-07T12:47:55.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 58 2024-02-07T12:47:55.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 460f5071-39a7-44b5-b635-e5923fb6e0d9 2024-02-07T12:47:55.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707310074813609, "job": 1, "event": "recovery_started", "wal_files": [58]} 2024-02-07T12:47:55.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #58 mode 2 2024-02-07T12:47:55.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707310074821468, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 63, "file_size": 2322302, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 26789, "largest_seqno": 29017, "table_properties": {"data_size": 2313273, "index_size": 5449, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 2565, "raw_key_size": 25059, "raw_average_key_size": 24, "raw_value_size": 2292812, "raw_average_value_size": 2279, "num_data_blocks": 241, "num_entries": 1006, "num_filter_entries": 1006, "num_deletions": 8, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1707310074, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "460f5071-39a7-44b5-b635-e5923fb6e0d9", "db_session_id": "IY0CMSS65G7P3S9LXC4H", "orig_file_number": 63, "seqno_to_time_mapping": "N/A"}} 2024-02-07T12:47:55.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707310074821575, "job": 1, "event": "recovery_finished"} 2024-02-07T12:47:55.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: [db/version_set.cc:5047] Creating manifest 65 2024-02-07T12:47:55.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-02-07T12:47:55.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000058.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-02-07T12:47:55.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5565ca3b8000 2024-02-07T12:47:55.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: DB pointer 0x5565ca3a2000 2024-02-07T12:47:55.114 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-02-07T12:47:55.114 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-02-07T12:47:55.114 INFO:journalctl@ceph.mon.a.smithi019.stdout: ** DB Stats ** 2024-02-07T12:47:55.114 INFO:journalctl@ceph.mon.a.smithi019.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-02-07T12:47:55.114 INFO:journalctl@ceph.mon.a.smithi019.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-02-07T12:47:55.114 INFO:journalctl@ceph.mon.a.smithi019.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-02-07T12:47:55.114 INFO:journalctl@ceph.mon.a.smithi019.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-07T12:47:55.114 INFO:journalctl@ceph.mon.a.smithi019.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-02-07T12:47:55.114 INFO:journalctl@ceph.mon.a.smithi019.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-02-07T12:47:55.114 INFO:journalctl@ceph.mon.a.smithi019.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-07T12:47:55.114 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2024-02-07T12:47:55.114 INFO:journalctl@ceph.mon.a.smithi019.stdout: ** Compaction Stats [default] ** 2024-02-07T12:47:55.114 INFO:journalctl@ceph.mon.a.smithi019.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-02-07T12:47:55.114 INFO:journalctl@ceph.mon.a.smithi019.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-02-07T12:47:55.114 INFO:journalctl@ceph.mon.a.smithi019.stdout: L0 1/0 2.21 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 581.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-02-07T12:47:55.115 INFO:journalctl@ceph.mon.a.smithi019.stdout: L6 1/0 8.81 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-02-07T12:47:55.115 INFO:journalctl@ceph.mon.a.smithi019.stdout: Sum 2/0 11.02 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 581.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-02-07T12:47:55.115 INFO:journalctl@ceph.mon.a.smithi019.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 581.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-02-07T12:47:55.115 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2024-02-07T12:47:55.115 INFO:journalctl@ceph.mon.a.smithi019.stdout: ** Compaction Stats [default] ** 2024-02-07T12:47:55.115 INFO:journalctl@ceph.mon.a.smithi019.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-02-07T12:47:55.115 INFO:journalctl@ceph.mon.a.smithi019.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-07T12:47:55.115 INFO:journalctl@ceph.mon.a.smithi019.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 581.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-02-07T12:47:55.115 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2024-02-07T12:47:55.115 INFO:journalctl@ceph.mon.a.smithi019.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-02-07T12:47:55.115 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2024-02-07T12:47:55.115 INFO:journalctl@ceph.mon.a.smithi019.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-02-07T12:47:55.115 INFO:journalctl@ceph.mon.a.smithi019.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-02-07T12:47:55.115 INFO:journalctl@ceph.mon.a.smithi019.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-02-07T12:47:55.116 INFO:journalctl@ceph.mon.a.smithi019.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-02-07T12:47:55.116 INFO:journalctl@ceph.mon.a.smithi019.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-02-07T12:47:55.116 INFO:journalctl@ceph.mon.a.smithi019.stdout: AddFile(Keys): cumulative 0, interval 0 2024-02-07T12:47:55.116 INFO:journalctl@ceph.mon.a.smithi019.stdout: Cumulative compaction: 0.00 GB write, 216.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:47:55.116 INFO:journalctl@ceph.mon.a.smithi019.stdout: Interval compaction: 0.00 GB write, 216.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:47:55.116 INFO:journalctl@ceph.mon.a.smithi019.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-02-07T12:47:55.116 INFO:journalctl@ceph.mon.a.smithi019.stdout: Block cache BinnedLRUCache@0x5565ca2c7090#2 capacity: 512.00 MB usage: 8.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-02-07T12:47:55.116 INFO:journalctl@ceph.mon.a.smithi019.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.78 KB,0.000530481%) IndexBlock(1,6.08 KB,0.00115931%) Misc(1,0.00 KB,0%) 2024-02-07T12:47:55.116 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2024-02-07T12:47:55.116 INFO:journalctl@ceph.mon.a.smithi019.stdout: ** File Read Latency Histogram By Level [default] ** 2024-02-07T12:47:55.116 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: starting mon.a rank 0 at public addrs [v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0] at bind addrs [v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:47:55.116 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: mon.a@-1(???) e3 preinit fsid ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:47:55.116 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: mon.a@-1(???).mds e1 new map 2024-02-07T12:47:55.116 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: mon.a@-1(???).mds e1 print_map 2024-02-07T12:47:55.117 INFO:journalctl@ceph.mon.a.smithi019.stdout: e1 2024-02-07T12:47:55.117 INFO:journalctl@ceph.mon.a.smithi019.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-02-07T12:47:55.117 INFO:journalctl@ceph.mon.a.smithi019.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-02-07T12:47:55.117 INFO:journalctl@ceph.mon.a.smithi019.stdout: legacy client fscid: -1 2024-02-07T12:47:55.117 INFO:journalctl@ceph.mon.a.smithi019.stdout: 2024-02-07T12:47:55.117 INFO:journalctl@ceph.mon.a.smithi019.stdout: No filesystems configured 2024-02-07T12:47:55.117 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: mon.a@-1(???).osd e104 crush map has features 3314933000854323200, adjusting msgr requires 2024-02-07T12:47:55.117 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: mon.a@-1(???).osd e104 crush map has features 432629239337189376, adjusting msgr requires 2024-02-07T12:47:55.117 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: mon.a@-1(???).osd e104 crush map has features 432629239337189376, adjusting msgr requires 2024-02-07T12:47:55.117 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: mon.a@-1(???).osd e104 crush map has features 432629239337189376, adjusting msgr requires 2024-02-07T12:47:55.117 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 ceph-mon[82918]: mon.a@-1(???).paxosservice(auth 1..26) refresh upgraded, format 0 -> 3 2024-02-07T12:47:55.117 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 bash[82899]: 8b1b0e21b1c548070914ac24243372f162013338f674502ce47637e51c54119d 2024-02-07T12:47:55.117 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:54 smithi019 systemd[1]: Started Ceph mon.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:47:58.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[82918]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:47:58.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[82918]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:47:58.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[82918]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:47:58.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[82918]: mon.a calling monitor election 2024-02-07T12:47:58.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[82918]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-02-07T12:47:58.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[82918]: monmap e3: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-02-07T12:47:58.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[82918]: fsmap 2024-02-07T12:47:58.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[82918]: osdmap e104: 8 total, 8 up, 8 in 2024-02-07T12:47:58.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[82918]: mgrmap e43: y(active, since 3m), standbys: x 2024-02-07T12:47:58.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[82918]: from='mgr.15297 ' entity='' 2024-02-07T12:47:58.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[82918]: mgrmap e44: y(active, since 3m), standbys: x 2024-02-07T12:47:58.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[82918]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-02-07T12:47:58.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[82918]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-02-07T12:47:58.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[82918]: daemon iscsi.foo.smithi019.svjimj on smithi019 is in unknown state 2024-02-07T12:47:58.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ignoring --setuser ceph since I am not root 2024-02-07T12:47:58.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ignoring --setgroup ceph since I am not root 2024-02-07T12:47:58.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mgr[61877]: -- 172.21.15.19:0/355312669 <== mon.1 v2:172.21.15.19:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5626a270b520 con 0x5626a26e1000 2024-02-07T12:47:58.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[29267]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:47:58.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[29267]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:47:58.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[29267]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:47:58.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[29267]: mon.a calling monitor election 2024-02-07T12:47:58.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[29267]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-02-07T12:47:58.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[29267]: monmap e3: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-02-07T12:47:58.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[29267]: fsmap 2024-02-07T12:47:58.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[29267]: osdmap e104: 8 total, 8 up, 8 in 2024-02-07T12:47:58.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[29267]: mgrmap e43: y(active, since 3m), standbys: x 2024-02-07T12:47:58.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[29267]: from='mgr.15297 ' entity='' 2024-02-07T12:47:58.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[29267]: mgrmap e44: y(active, since 3m), standbys: x 2024-02-07T12:47:58.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[29267]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-02-07T12:47:58.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[29267]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-02-07T12:47:58.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:57 smithi019 ceph-mon[29267]: daemon iscsi.foo.smithi019.svjimj on smithi019 is in unknown state 2024-02-07T12:47:58.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:47:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: ignoring --setuser ceph since I am not root 2024-02-07T12:47:58.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:47:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: ignoring --setgroup ceph since I am not root 2024-02-07T12:47:58.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:47:57 smithi172 ceph-mgr[60839]: -- 172.21.15.172:0/3360423530 <== mon.2 v2:172.21.15.172:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55870fd51520 con 0x55870fd27000 2024-02-07T12:47:58.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:57 smithi172 ceph-mon[67636]: from='mgr.15297 172.21.15.19:0/3106209013' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:47:58.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:57 smithi172 ceph-mon[67636]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:47:58.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:57 smithi172 ceph-mon[67636]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:47:58.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:57 smithi172 ceph-mon[67636]: mon.a calling monitor election 2024-02-07T12:47:58.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:57 smithi172 ceph-mon[67636]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-02-07T12:47:58.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:57 smithi172 ceph-mon[67636]: monmap e3: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-02-07T12:47:58.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:57 smithi172 ceph-mon[67636]: fsmap 2024-02-07T12:47:58.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:57 smithi172 ceph-mon[67636]: osdmap e104: 8 total, 8 up, 8 in 2024-02-07T12:47:58.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:57 smithi172 ceph-mon[67636]: mgrmap e43: y(active, since 3m), standbys: x 2024-02-07T12:47:58.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:57 smithi172 ceph-mon[67636]: from='mgr.15297 ' entity='' 2024-02-07T12:47:58.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:57 smithi172 ceph-mon[67636]: mgrmap e44: y(active, since 3m), standbys: x 2024-02-07T12:47:58.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:57 smithi172 ceph-mon[67636]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-02-07T12:47:58.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:57 smithi172 ceph-mon[67636]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-02-07T12:47:58.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:57 smithi172 ceph-mon[67636]: daemon iscsi.foo.smithi019.svjimj on smithi019 is in unknown state 2024-02-07T12:47:58.844 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:58 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:47:58.464+0000 7fb796cd5200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-02-07T12:47:58.844 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:58 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:47:58.559+0000 7fb796cd5200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-02-07T12:47:58.869 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:47:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:47:58.452+0000 7fb3751cb200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-02-07T12:47:58.869 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:47:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:47:58.546+0000 7fb3751cb200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-02-07T12:47:59.530 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:47:59 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:47:59.194+0000 7fb3751cb200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-02-07T12:47:59.530 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:47:59 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:47:59.287+0000 7fb3751cb200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-02-07T12:47:59.531 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:47:59 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:47:59.431+0000 7fb3751cb200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-02-07T12:47:59.531 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:47:59 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:47:59.528+0000 7fb3751cb200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-02-07T12:47:59.559 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:59 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:47:59.220+0000 7fb796cd5200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-02-07T12:47:59.559 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:59 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:47:59.315+0000 7fb796cd5200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-02-07T12:47:59.559 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:59 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:47:59.459+0000 7fb796cd5200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-02-07T12:47:59.560 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:59 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:47:59.557+0000 7fb796cd5200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-02-07T12:47:59.844 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:59 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:47:59.643+0000 7fb796cd5200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-02-07T12:47:59.869 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:47:59 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:47:59.613+0000 7fb3751cb200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-02-07T12:48:00.177 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:47:59 smithi172 ceph-mon[67636]: mgrmap e45: y(active, since 3m), standbys: x 2024-02-07T12:48:00.178 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:47:59 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:47:59.881+0000 7fb3751cb200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-02-07T12:48:00.178 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:47:59 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:47:59.992+0000 7fb3751cb200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-02-07T12:48:00.178 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:48:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:48:00.079+0000 7fb3751cb200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-02-07T12:48:00.178 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:48:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:48:00.175+0000 7fb3751cb200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-02-07T12:48:00.217 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:47:59 smithi019 ceph-mon[29267]: mgrmap e45: y(active, since 3m), standbys: x 2024-02-07T12:48:00.217 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:47:59 smithi019 ceph-mon[82918]: mgrmap e45: y(active, since 3m), standbys: x 2024-02-07T12:48:00.217 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:47:59 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:47:59.917+0000 7fb796cd5200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-02-07T12:48:00.217 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:48:00.029+0000 7fb796cd5200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-02-07T12:48:00.217 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:48:00.118+0000 7fb796cd5200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-02-07T12:48:00.218 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:48:00.215+0000 7fb796cd5200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-02-07T12:48:00.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:00 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:48:00.395+0000 7fb796cd5200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-02-07T12:48:00.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:48:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:48:00.355+0000 7fb3751cb200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-02-07T12:48:01.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:48:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:48:01.329+0000 7fb3751cb200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-02-07T12:48:01.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:48:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:48:01.462+0000 7fb3751cb200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-02-07T12:48:01.844 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:01 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:48:01.389+0000 7fb796cd5200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-02-07T12:48:01.844 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:01 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:48:01.524+0000 7fb796cd5200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-02-07T12:48:02.103 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:48:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:48:01.808+0000 7fb3751cb200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-02-07T12:48:02.103 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:48:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:48:01.901+0000 7fb3751cb200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-02-07T12:48:02.103 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:48:02 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:48:02.101+0000 7fb3751cb200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:48:02.167 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:01 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:48:01.869+0000 7fb796cd5200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-02-07T12:48:02.167 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:01 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:48:01.962+0000 7fb796cd5200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-02-07T12:48:02.168 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:02 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:48:02.165+0000 7fb796cd5200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:48:02.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:48:02 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:48:02.293+0000 7fb3751cb200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-02-07T12:48:02.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:02 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:48:02.357+0000 7fb796cd5200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-02-07T12:48:02.856 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:48:02 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:48:02.588+0000 7fb3751cb200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:48:02.924 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:02 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:48:02.656+0000 7fb796cd5200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-02-07T12:48:03.119 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:48:02 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:48:02.854+0000 7fb3751cb200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-02-07T12:48:03.119 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:48:03 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:48:03.096+0000 7fb3751cb200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-02-07T12:48:03.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:02 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:48:02.923+0000 7fb796cd5200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-02-07T12:48:03.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:48:03.170+0000 7fb796cd5200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-02-07T12:48:03.844 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:48:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:48:03.514Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi019.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.19:8443: connect: connection refused" 2024-02-07T12:48:03.844 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:48:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:48:03.514Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi172.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.172:8443: connect: connection refused" 2024-02-07T12:48:04.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:04 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:48:04.011+0000 7fb796cd5200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-02-07T12:48:04.344 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:48:04 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:48:04.242Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi019.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.19:8443: connect: connection refused" 2024-02-07T12:48:04.344 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:48:04 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:48:04.242Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi172.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.172:8443: connect: connection refused" 2024-02-07T12:48:04.369 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:48:03 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:48:03.923+0000 7fb3751cb200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-02-07T12:48:04.844 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:48:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:48:04.510+0000 7fb3751cb200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-02-07T12:48:04.844 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:48:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:48:04.613+0000 7fb3751cb200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-02-07T12:48:04.844 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:48:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:48:04.842+0000 7fb3751cb200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-02-07T12:48:04.943 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:04 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:48:04.604+0000 7fb796cd5200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-02-07T12:48:04.943 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:04 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:48:04.710+0000 7fb796cd5200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-02-07T12:48:05.119 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:48:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: [07/Feb/2024:12:48:04] ENGINE Bus STARTING 2024-02-07T12:48:05.119 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:48:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: CherryPy Checker: 2024-02-07T12:48:05.119 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:48:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: The Application mounted at '' has an empty config. 2024-02-07T12:48:05.119 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:48:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: 2024-02-07T12:48:05.119 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:48:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: [07/Feb/2024:12:48:04] ENGINE Serving on http://:::9283 2024-02-07T12:48:05.120 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:48:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x[60835]: [07/Feb/2024:12:48:04] ENGINE Bus STARTED 2024-02-07T12:48:05.197 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:04 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-c[29262]: 2024-02-07T12:48:04.961+0000 7fa087799700 -1 mon.c@1(peon).osd e105 update_from_paxos full map CRC mismatch, resetting to canonical 2024-02-07T12:48:05.197 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:04 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:48:04.941+0000 7fb796cd5200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-02-07T12:48:05.198 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: [07/Feb/2024:12:48:05] ENGINE Bus STARTING 2024-02-07T12:48:05.594 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:48:05.251Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=3 2024-02-07T12:48:05.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CherryPy Checker: 2024-02-07T12:48:05.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: The Application mounted at '' has an empty config. 2024-02-07T12:48:05.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:48:05.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: [07/Feb/2024:12:48:05] ENGINE Serving on http://:::9283 2024-02-07T12:48:05.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: [07/Feb/2024:12:48:05] ENGINE Bus STARTED 2024-02-07T12:48:05.985 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:48:05.728Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=4 2024-02-07T12:48:05.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: Standby manager daemon x restarted 2024-02-07T12:48:05.986 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: Standby manager daemon x started 2024-02-07T12:48:05.987 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/161060461' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-02-07T12:48:05.987 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/161060461' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:48:05.987 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/161060461' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-02-07T12:48:05.987 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: from='mgr.? 172.21.15.172:0/161060461' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:48:05.987 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: Active manager daemon y restarted 2024-02-07T12:48:05.987 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: Activating manager daemon y 2024-02-07T12:48:05.987 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: failed to encode map e105 with expected crc 2024-02-07T12:48:05.987 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: osdmap e105: 8 total, 8 up, 8 in 2024-02-07T12:48:05.987 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: mgrmap e46: y(active, starting, since 0.0104789s), standbys: x 2024-02-07T12:48:05.987 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:48:05.987 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:48:05.987 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:48:05.987 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: failed to encode map e105 with expected crc 2024-02-07T12:48:05.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-02-07T12:48:05.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-02-07T12:48:05.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:48:05.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:48:05.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:48:05.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:48:05.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:48:05.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:48:05.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:48:05.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:48:05.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:48:05.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:48:05.988 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:48:05.989 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: Manager daemon y is now available 2024-02-07T12:48:05.989 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:48:05.989 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:48:05.989 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:48:05.989 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:48:05.989 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:48:05.989 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: failed to encode map e105 with expected crc 2024-02-07T12:48:05.989 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: failed to encode map e105 with expected crc 2024-02-07T12:48:05.989 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:48:05.989 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[29267]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:48:05.990 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: Standby manager daemon x restarted 2024-02-07T12:48:05.990 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: Standby manager daemon x started 2024-02-07T12:48:05.990 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: from='mgr.? 172.21.15.172:0/161060461' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-02-07T12:48:05.990 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: from='mgr.? 172.21.15.172:0/161060461' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:48:05.990 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: from='mgr.? 172.21.15.172:0/161060461' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-02-07T12:48:05.990 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: from='mgr.? 172.21.15.172:0/161060461' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:48:05.990 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: Active manager daemon y restarted 2024-02-07T12:48:05.990 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: Activating manager daemon y 2024-02-07T12:48:05.990 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: failed to encode map e105 with expected crc 2024-02-07T12:48:05.990 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: osdmap e105: 8 total, 8 up, 8 in 2024-02-07T12:48:05.991 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: mgrmap e46: y(active, starting, since 0.0104789s), standbys: x 2024-02-07T12:48:05.991 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:48:05.991 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:48:05.991 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:48:05.991 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: failed to encode map e105 with expected crc 2024-02-07T12:48:05.991 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-02-07T12:48:05.991 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-02-07T12:48:05.991 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:48:05.991 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:48:05.991 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:48:05.991 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:48:05.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:48:05.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:48:05.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:48:05.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:48:05.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:48:05.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:48:05.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:48:05.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: Manager daemon y is now available 2024-02-07T12:48:05.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:48:05.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:48:05.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:48:05.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:48:05.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:48:05.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: failed to encode map e105 with expected crc 2024-02-07T12:48:05.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: failed to encode map e105 with expected crc 2024-02-07T12:48:05.993 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:48:05.993 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:48:06.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: Standby manager daemon x restarted 2024-02-07T12:48:06.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: Standby manager daemon x started 2024-02-07T12:48:06.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: from='mgr.? 172.21.15.172:0/161060461' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-02-07T12:48:06.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: from='mgr.? 172.21.15.172:0/161060461' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:48:06.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: from='mgr.? 172.21.15.172:0/161060461' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-02-07T12:48:06.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: from='mgr.? 172.21.15.172:0/161060461' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:48:06.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: Active manager daemon y restarted 2024-02-07T12:48:06.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: Activating manager daemon y 2024-02-07T12:48:06.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: failed to encode map e105 with expected crc 2024-02-07T12:48:06.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: osdmap e105: 8 total, 8 up, 8 in 2024-02-07T12:48:06.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: mgrmap e46: y(active, starting, since 0.0104789s), standbys: x 2024-02-07T12:48:06.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:48:06.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:48:06.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:48:06.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: failed to encode map e105 with expected crc 2024-02-07T12:48:06.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-02-07T12:48:06.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-02-07T12:48:06.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:48:06.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:48:06.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:48:06.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:48:06.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:48:06.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:48:06.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:48:06.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:48:06.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:48:06.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:48:06.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:48:06.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: Manager daemon y is now available 2024-02-07T12:48:06.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:48:06.121 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:48:06.122 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:48:06.122 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:48:06.122 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:48:06.122 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: failed to encode map e105 with expected crc 2024-02-07T12:48:06.122 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: failed to encode map e105 with expected crc 2024-02-07T12:48:06.122 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:48:06.122 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-02-07T12:48:06.262 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:05 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:48:05.983+0000 7fb74ced9700 -1 mgr.server handle_report got status from non-daemon mon.a 2024-02-07T12:48:06.519 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:48:06 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:48:06.260Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=4 2024-02-07T12:48:06.519 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:48:06 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:48:06.518Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=4 2024-02-07T12:48:07.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:06 smithi019 ceph-mon[82918]: [07/Feb/2024:12:48:05] ENGINE Bus STARTING 2024-02-07T12:48:07.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:06 smithi019 ceph-mon[82918]: [07/Feb/2024:12:48:05] ENGINE Serving on https://172.21.15.19:7150 2024-02-07T12:48:07.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:06 smithi019 ceph-mon[82918]: [07/Feb/2024:12:48:05] ENGINE Serving on http://172.21.15.19:8765 2024-02-07T12:48:07.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:06 smithi019 ceph-mon[82918]: [07/Feb/2024:12:48:05] ENGINE Bus STARTED 2024-02-07T12:48:07.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:06 smithi019 ceph-mon[82918]: failed to encode map e105 with expected crc 2024-02-07T12:48:07.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:06 smithi019 ceph-mon[82918]: mgrmap e47: y(active, since 1.02837s), standbys: x 2024-02-07T12:48:07.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:06 smithi019 ceph-mon[29267]: [07/Feb/2024:12:48:05] ENGINE Bus STARTING 2024-02-07T12:48:07.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:06 smithi019 ceph-mon[29267]: [07/Feb/2024:12:48:05] ENGINE Serving on https://172.21.15.19:7150 2024-02-07T12:48:07.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:06 smithi019 ceph-mon[29267]: [07/Feb/2024:12:48:05] ENGINE Serving on http://172.21.15.19:8765 2024-02-07T12:48:07.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:06 smithi019 ceph-mon[29267]: [07/Feb/2024:12:48:05] ENGINE Bus STARTED 2024-02-07T12:48:07.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:06 smithi019 ceph-mon[29267]: failed to encode map e105 with expected crc 2024-02-07T12:48:07.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:06 smithi019 ceph-mon[29267]: mgrmap e47: y(active, since 1.02837s), standbys: x 2024-02-07T12:48:07.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:06 smithi172 ceph-mon[67636]: [07/Feb/2024:12:48:05] ENGINE Bus STARTING 2024-02-07T12:48:07.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:06 smithi172 ceph-mon[67636]: [07/Feb/2024:12:48:05] ENGINE Serving on https://172.21.15.19:7150 2024-02-07T12:48:07.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:06 smithi172 ceph-mon[67636]: [07/Feb/2024:12:48:05] ENGINE Serving on http://172.21.15.19:8765 2024-02-07T12:48:07.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:06 smithi172 ceph-mon[67636]: [07/Feb/2024:12:48:05] ENGINE Bus STARTED 2024-02-07T12:48:07.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:06 smithi172 ceph-mon[67636]: failed to encode map e105 with expected crc 2024-02-07T12:48:07.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:06 smithi172 ceph-mon[67636]: mgrmap e47: y(active, since 1.02837s), standbys: x 2024-02-07T12:48:08.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:07 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:48:08.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:07 smithi019 ceph-mon[29267]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:48:08.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:07 smithi019 ceph-mon[29267]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-02-07T12:48:08.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:07 smithi019 ceph-mon[29267]: Cluster is now healthy 2024-02-07T12:48:08.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:07 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:48:08.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:07 smithi019 ceph-mon[82918]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:48:08.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:07 smithi019 ceph-mon[82918]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-02-07T12:48:08.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:07 smithi019 ceph-mon[82918]: Cluster is now healthy 2024-02-07T12:48:08.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:07 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:48:08.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:07 smithi172 ceph-mon[67636]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:48:08.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:07 smithi172 ceph-mon[67636]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-02-07T12:48:08.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:07 smithi172 ceph-mon[67636]: Cluster is now healthy 2024-02-07T12:48:09.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:08 smithi019 ceph-mon[29267]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:48:09.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:08 smithi019 ceph-mon[29267]: mgrmap e48: y(active, since 2s), standbys: x 2024-02-07T12:48:09.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:08 smithi019 ceph-mon[82918]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:48:09.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:08 smithi019 ceph-mon[82918]: mgrmap e48: y(active, since 2s), standbys: x 2024-02-07T12:48:09.097 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:08 smithi172 ceph-mon[67636]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:48:09.097 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:08 smithi172 ceph-mon[67636]: mgrmap e48: y(active, since 2s), standbys: x 2024-02-07T12:48:11.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:10 smithi019 ceph-mon[29267]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:48:11.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:10 smithi019 ceph-mon[29267]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:11.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:10 smithi019 ceph-mon[29267]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:11.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:10 smithi019 ceph-mon[29267]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:11.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:10 smithi019 ceph-mon[29267]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:11.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:10 smithi019 ceph-mon[29267]: mgrmap e49: y(active, since 4s), standbys: x 2024-02-07T12:48:11.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:10 smithi019 ceph-mon[82918]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:48:11.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:10 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:11.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:10 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:11.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:10 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:11.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:10 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:11.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:10 smithi019 ceph-mon[82918]: mgrmap e49: y(active, since 4s), standbys: x 2024-02-07T12:48:11.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:10 smithi172 ceph-mon[67636]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:48:11.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:10 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:11.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:10 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:11.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:10 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:11.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:10 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:11.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:10 smithi172 ceph-mon[67636]: mgrmap e49: y(active, since 4s), standbys: x 2024-02-07T12:48:12.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:11 smithi019 ceph-mon[29267]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:12.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:11 smithi019 ceph-mon[29267]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:12.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:11 smithi019 ceph-mon[29267]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:12.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:11 smithi019 ceph-mon[29267]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:12.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:11 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:12.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:11 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:12.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:11 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:12.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:11 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:12.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:11 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:12.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:11 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:12.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:11 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:12.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:11 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:12.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:12 smithi019 ceph-mon[82918]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:48:12.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:12 smithi019 ceph-mon[29267]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:48:13.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:12 smithi172 ceph-mon[67636]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:48:14.000 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:48:13] "GET /metrics HTTP/1.1" 200 35204 "" "Prometheus/2.43.0" 2024-02-07T12:48:15.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:14 smithi019 ceph-mon[29267]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-02-07T12:48:15.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:14 smithi019 ceph-mon[29267]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:15.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:14 smithi019 ceph-mon[29267]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:15.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:14 smithi019 ceph-mon[29267]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:15.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:14 smithi019 ceph-mon[29267]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:15.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:14 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:15.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:14 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:48:15.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:14 smithi019 ceph-mon[82918]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-02-07T12:48:15.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:14 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:15.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:14 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:15.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:14 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:15.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:14 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:15.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:14 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:15.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:14 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:48:15.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:14 smithi172 ceph-mon[67636]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-02-07T12:48:15.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:14 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:15.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:14 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:15.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:14 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:15.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:14 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:15.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:14 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:15.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:14 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:48:16.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[29267]: Detected new or changed devices on smithi019 2024-02-07T12:48:16.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[29267]: Updating smithi019:/etc/ceph/ceph.conf 2024-02-07T12:48:16.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[29267]: Updating smithi172:/etc/ceph/ceph.conf 2024-02-07T12:48:16.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[29267]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:48:16.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[29267]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:48:16.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[29267]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:48:16.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[29267]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:48:16.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[29267]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:16.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[29267]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:16.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[29267]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:16.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[29267]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:16.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[29267]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:16.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:48:16.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[29267]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:48:16.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:16.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[82918]: Detected new or changed devices on smithi019 2024-02-07T12:48:16.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[82918]: Updating smithi019:/etc/ceph/ceph.conf 2024-02-07T12:48:16.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[82918]: Updating smithi172:/etc/ceph/ceph.conf 2024-02-07T12:48:16.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[82918]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:48:16.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[82918]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:48:16.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[82918]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:48:16.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[82918]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:48:16.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:16.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:16.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:16.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:16.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:16.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:48:16.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:48:16.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:15 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:16.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:15 smithi172 ceph-mon[67636]: Detected new or changed devices on smithi019 2024-02-07T12:48:16.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:15 smithi172 ceph-mon[67636]: Updating smithi019:/etc/ceph/ceph.conf 2024-02-07T12:48:16.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:15 smithi172 ceph-mon[67636]: Updating smithi172:/etc/ceph/ceph.conf 2024-02-07T12:48:16.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:15 smithi172 ceph-mon[67636]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:48:16.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:15 smithi172 ceph-mon[67636]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:48:16.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:15 smithi172 ceph-mon[67636]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:48:16.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:15 smithi172 ceph-mon[67636]: Updating smithi019:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:48:16.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:15 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:16.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:15 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:16.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:15 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:16.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:15 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:16.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:15 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:16.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:15 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:48:16.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:15 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:48:16.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:15 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:17.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:16 smithi172 ceph-mon[67636]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:48:17.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:16 smithi172 ceph-mon[67636]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:48:17.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:16 smithi172 ceph-mon[67636]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-02-07T12:48:17.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:16 smithi172 ceph-mon[67636]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-02-07T12:48:17.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:16 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:48:17.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:16 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:48:17.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:16 smithi172 ceph-mon[67636]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-02-07T12:48:17.120 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:16 smithi019 ceph-mon[29267]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:48:17.120 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:16 smithi019 ceph-mon[29267]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:48:17.120 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:16 smithi019 ceph-mon[29267]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-02-07T12:48:17.121 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:16 smithi019 ceph-mon[29267]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-02-07T12:48:17.121 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:16 smithi019 ceph-mon[29267]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:48:17.121 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:16 smithi019 ceph-mon[29267]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:48:17.121 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:16 smithi019 ceph-mon[29267]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-02-07T12:48:17.121 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:16 smithi019 ceph-mon[82918]: Updating smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:48:17.121 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:16 smithi019 ceph-mon[82918]: Updating smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:48:17.121 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:16 smithi019 ceph-mon[82918]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-02-07T12:48:17.121 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:16 smithi019 ceph-mon[82918]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-02-07T12:48:17.121 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:16 smithi019 ceph-mon[82918]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:48:17.121 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:16 smithi019 ceph-mon[82918]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:48:17.121 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:16 smithi019 ceph-mon[82918]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-02-07T12:48:17.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:48:17.199+0000 7fb71d671700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:48:17.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:48:17.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:48:17.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:48:17.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:48:17.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:17.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:17.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:48:17.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:48:17.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:48:17.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:48:17.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:48:17.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:48:17.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:48:17.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:48:17.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:48:17.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:48:17.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:48:17.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:48:17.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:17.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:17.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:48:17.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:48:17.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:48:17.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:48:17.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:48:17.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:48:17.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:48:17.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:48:17.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:48:17.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:48:17.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:48:17.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:48:17.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:48:17.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:48:17.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:48:17.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:48:17.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:48:17.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:48:17.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:48:17.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:48:17.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:48:17.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:48:17.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:17.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:17.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:48:17.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:48:17.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:48:17.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:48:17.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:48:17.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:48:17.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:48:17.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:48:17.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:48:17.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:48:17.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:48:17.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:48:17.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:17.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:18.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-mon[29267]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:48:18.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:48:18.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:48:18.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-02-07T12:48:18.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-02-07T12:48:18.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:48:18.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:48:18.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:48:18.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-02-07T12:48:18.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:17 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-02-07T12:48:18.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:17 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:48:18.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:17 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:48:18.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:17 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:48:18.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:17 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-02-07T12:48:18.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:17 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-02-07T12:48:19.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:18 smithi019 ceph-mon[82918]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:48:19.094 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:48:19.094 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:48:19.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:48:19.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:48:19.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:19.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:19.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:48:19.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:48:19.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:48:19.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:48:19.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:48:19.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:48:19.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:48:19.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:48:19.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:48:19.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:48:19.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:48:19.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:48:19.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:19.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:19.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:48:19.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:48:19.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:48:19.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:48:19.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:48:19.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:48:19.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:48:19.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:48:19.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:48:19.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:48:19.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:48:19.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:48:19.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:48:19.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:48:19.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:48:19.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:48:19.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:48:19.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:48:19.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:48:19.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:48:19.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:48:19.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:48:19.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:19.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:19.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:48:19.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:48:19.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:48:19.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:48:19.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:48:19.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:48:19.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:48:19.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:48:19.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:48:19.098 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:18 smithi019 ceph-mon[29267]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:48:19.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:48:19.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:48:19.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:48:19.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:48:19.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:19.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:19.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:48:19.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:48:19.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:48:19.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:48:19.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:48:19.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:48:19.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:48:19.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:48:19.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:48:19.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:48:19.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:48:19.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:48:19.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:19.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:19.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:48:19.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:48:19.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:48:19.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:48:19.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:48:19.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:48:19.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:48:19.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:48:19.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:48:19.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:48:19.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:48:19.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:48:19.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:48:19.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:48:19.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:48:19.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:48:19.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:48:19.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:48:19.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:48:19.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:48:19.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:48:19.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:48:19.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:19.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:19.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:48:19.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:48:19.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:48:19.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:48:19.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:48:19.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:48:19.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:48:19.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:48:19.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:48:19.102 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:48:19.102 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:48:19.103 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:48:19.103 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:19.103 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:19.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:18 smithi019 ceph-mon[82918]: Upgrade: It appears safe to stop mon.c 2024-02-07T12:48:19.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:18 smithi019 ceph-mon[82918]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-02-07T12:48:19.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:18 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:19.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:18 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:48:19.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:18 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:48:19.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:18 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:19.103 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:48:19.104 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:48:19.104 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:48:19.104 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:19.104 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:19.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:18 smithi019 ceph-mon[29267]: Upgrade: It appears safe to stop mon.c 2024-02-07T12:48:19.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:18 smithi019 ceph-mon[29267]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-02-07T12:48:19.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:18 smithi019 ceph-mon[29267]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:19.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:18 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:48:19.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:18 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:48:19.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:18 smithi019 ceph-mon[29267]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:19.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:18 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:48:19.119 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:48:19.119 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:48:19.119 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:48:19.119 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:48:19.119 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:19.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:19.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:48:19.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:48:19.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:48:19.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:48:19.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:48:19.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:48:19.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:48:19.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:48:19.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:48:19.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:48:19.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:48:19.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:48:19.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:19.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:19.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:48:19.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:48:19.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:48:19.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:48:19.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:48:19.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:48:19.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:48:19.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:48:19.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:48:19.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:48:19.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:48:19.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:48:19.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:48:19.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:48:19.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:48:19.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:48:19.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:48:19.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:48:19.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:48:19.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:48:19.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:48:19.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:48:19.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:19.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:19.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:48:19.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:48:19.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:48:19.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:48:19.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:48:19.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:48:19.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:48:19.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:48:19.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:48:19.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:48:19.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:48:19.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:48:19.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:19.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:19.123 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:18 smithi172 ceph-mon[67636]: Upgrade: It appears safe to stop mon.c 2024-02-07T12:48:19.123 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:18 smithi172 ceph-mon[67636]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-02-07T12:48:19.123 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:18 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:19.123 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:18 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:48:19.123 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:18 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:48:19.123 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:18 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:20.044 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:48:20.045 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:19 smithi019 ceph-mon[29267]: Upgrade: Updating mon.c 2024-02-07T12:48:20.045 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:19 smithi019 ceph-mon[29267]: Deploying daemon mon.c on smithi019 2024-02-07T12:48:20.045 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:19 smithi019 ceph-mon[82918]: Upgrade: Updating mon.c 2024-02-07T12:48:20.045 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:19 smithi019 ceph-mon[82918]: Deploying daemon mon.c on smithi019 2024-02-07T12:48:20.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:19 smithi172 ceph-mon[67636]: Upgrade: Updating mon.c 2024-02-07T12:48:20.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:19 smithi172 ceph-mon[67636]: Deploying daemon mon.c on smithi019 2024-02-07T12:48:20.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:20 smithi019 systemd[1]: Stopping Ceph mon.c for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:48:20.749 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:48:20.749 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (2m) 11s ago 24m 19.6M - 0.25.0 c8568f914cd2 fec9255f35e7 2024-02-07T12:48:20.749 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (2m) 11s ago 24m 82.9M - 9.4.12 a0ea5e3c46d9 aaffac63bc28 2024-02-07T12:48:20.749 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 unknown 11s ago 23m 60.6M - 3.5 e1d6a67b021e 5113aa77e3f2 2024-02-07T12:48:20.749 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283,8765 running (3m) 11s ago 28m 447M - 19.0.0-1252-g44458cce 79957b372307 5aa0c04fdfd7 2024-02-07T12:48:20.749 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (13m) 11s ago 30m 473M - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:48:20.749 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (26s) 11s ago 30m 34.2M 2048M 19.0.0-1252-g44458cce 79957b372307 8b1b0e21b1c5 2024-02-07T12:48:20.749 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (75s) 11s ago 29m 37.0M 2048M 19.0.0-1252-g44458cce 79957b372307 00f454bbac01 2024-02-07T12:48:20.749 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (29m) 11s ago 29m 70.0M 2048M 17.2.0 e1d6a67b021e b9825d0db32d 2024-02-07T12:48:20.749 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (3m) 11s ago 25m 8497k - 1.5.0 0da6a335fe13 c4bdcb1c4d49 2024-02-07T12:48:20.749 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (3m) 11s ago 25m 16.3M - 1.5.0 0da6a335fe13 04ea81f79d85 2024-02-07T12:48:20.749 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (28m) 11s ago 28m 54.4M 2993M 17.2.0 e1d6a67b021e 87f2560a9b15 2024-02-07T12:48:20.749 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (28m) 11s ago 28m 57.2M 2993M 17.2.0 e1d6a67b021e 0f9a190f0e0d 2024-02-07T12:48:20.750 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (27m) 11s ago 27m 52.8M 2993M 17.2.0 e1d6a67b021e 7db94f25a1aa 2024-02-07T12:48:20.750 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (27m) 11s ago 27m 53.4M 2993M 17.2.0 e1d6a67b021e 0b6b674188ed 2024-02-07T12:48:20.750 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (27m) 11s ago 27m 55.7M 3248M 17.2.0 e1d6a67b021e df72cfd69b3b 2024-02-07T12:48:20.750 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (26m) 11s ago 26m 52.4M 3248M 17.2.0 e1d6a67b021e 3bdb22c07537 2024-02-07T12:48:20.750 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (26m) 11s ago 26m 52.1M 3248M 17.2.0 e1d6a67b021e 7859b651e0d9 2024-02-07T12:48:20.750 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (25m) 11s ago 25m 55.5M 3248M 17.2.0 e1d6a67b021e b301e7c75d73 2024-02-07T12:48:20.750 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (2m) 11s ago 25m 61.0M - 2.43.0 a07b618ecd1d c9b607eea1be 2024-02-07T12:48:20.750 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (23m) 11s ago 23m 78.7M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:48:20.750 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (23m) 11s ago 23m 78.3M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:48:21.013 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:20 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-c[29262]: 2024-02-07T12:48:20.767+0000 7fa08cfa4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:48:21.013 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:20 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-c[29262]: 2024-02-07T12:48:20.767+0000 7fa08cfa4700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2024-02-07T12:48:21.158 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:48:21.159 INFO:teuthology.orchestra.run.smithi019.stdout: "mon": { 2024-02-07T12:48:21.159 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-02-07T12:48:21.159 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:48:21.159 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:48:21.159 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr": { 2024-02-07T12:48:21.159 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:48:21.159 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:48:21.159 INFO:teuthology.orchestra.run.smithi019.stdout: "osd": { 2024-02-07T12:48:21.159 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-02-07T12:48:21.159 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:48:21.159 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw": { 2024-02-07T12:48:21.159 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:48:21.159 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:48:21.159 INFO:teuthology.orchestra.run.smithi019.stdout: "overall": { 2024-02-07T12:48:21.160 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-02-07T12:48:21.160 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 4 2024-02-07T12:48:21.160 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:48:21.160 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:48:21.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:21 smithi019 podman[87927]: 2024-02-07 12:48:21.011762299 +0000 UTC m=+0.341059167 container died b9825d0db32d69a488d2490c8c4369dd4437443423031401a55d58061805812b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-c, RELEASE=HEAD, io.buildah.version=1.19.8, GIT_CLEAN=True, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, ceph=True, com.redhat.component=centos-stream-container, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-02-07T12:48:21.500 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:48:21.500 INFO:teuthology.orchestra.run.smithi019.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:48:21.500 INFO:teuthology.orchestra.run.smithi019.stdout: "in_progress": true, 2024-02-07T12:48:21.500 INFO:teuthology.orchestra.run.smithi019.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi019", 2024-02-07T12:48:21.500 INFO:teuthology.orchestra.run.smithi019.stdout: "services_complete": [], 2024-02-07T12:48:21.500 INFO:teuthology.orchestra.run.smithi019.stdout: "progress": "1/2 daemons upgraded", 2024-02-07T12:48:21.500 INFO:teuthology.orchestra.run.smithi019.stdout: "message": "Currently upgrading mon daemons", 2024-02-07T12:48:21.500 INFO:teuthology.orchestra.run.smithi019.stdout: "is_paused": false 2024-02-07T12:48:21.500 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:48:21.746 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:21 smithi019 podman[87927]: 2024-02-07 12:48:21.476658777 +0000 UTC m=+0.805955651 container cleanup b9825d0db32d69a488d2490c8c4369dd4437443423031401a55d58061805812b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-c, RELEASE=HEAD, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, vcs-type=git, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, version=8) 2024-02-07T12:48:21.746 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:21 smithi019 bash[87927]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-c 2024-02-07T12:48:22.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:21 smithi019 podman[87964]: 2024-02-07 12:48:21.744647054 +0000 UTC m=+0.734582670 container remove b9825d0db32d69a488d2490c8c4369dd4437443423031401a55d58061805812b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-c, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, GIT_CLEAN=True, vcs-type=git, distribution-scope=public, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, ceph=True, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-02-07T12:48:22.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:21 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mon.c.service: Deactivated successfully. 2024-02-07T12:48:22.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:21 smithi019 systemd[1]: Stopped Ceph mon.c for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:48:22.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:21 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mon.c.service: Consumed 16.893s CPU time. 2024-02-07T12:48:22.432 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 systemd[1]: Starting Ceph mon.c for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:48:22.432 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 podman[88119]: 2024-02-07 12:48:22.209729648 +0000 UTC m=+0.018959033 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:48:22.432 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 podman[88119]: 2024-02-07 12:48:22.329258231 +0000 UTC m=+0.138487607 container create 6214003f6e08b6bc4e7871cf9b636365217102d9b7431aadb7f35f80718c4d9d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-c, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-02-07T12:48:22.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 podman[88119]: 2024-02-07 12:48:22.557104088 +0000 UTC m=+0.366333483 container init 6214003f6e08b6bc4e7871cf9b636365217102d9b7431aadb7f35f80718c4d9d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-c, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, io.buildah.version=1.33.2, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.build-date=20240131, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-02-07T12:48:22.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 podman[88119]: 2024-02-07 12:48:22.560437911 +0000 UTC m=+0.369667288 container start 6214003f6e08b6bc4e7871cf9b636365217102d9b7431aadb7f35f80718c4d9d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-c, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-02-07T12:48:22.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: set uid:gid to 167:167 (ceph:ceph) 2024-02-07T12:48:22.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev), process ceph-mon, pid 2 2024-02-07T12:48:22.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: pidfile_write: ignore empty --pid-file 2024-02-07T12:48:22.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: load: jerasure load: lrc 2024-02-07T12:48:22.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: RocksDB version: 7.9.2 2024-02-07T12:48:22.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Git sha 0 2024-02-07T12:48:22.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Compile date 2024-02-05 19:41:51 2024-02-07T12:48:22.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: DB SUMMARY 2024-02-07T12:48:22.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: DB Session ID: PBA6UCWZMIUGOTNPMDJW 2024-02-07T12:48:22.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: CURRENT file: CURRENT 2024-02-07T12:48:22.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: IDENTITY file: IDENTITY 2024-02-07T12:48:22.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: MANIFEST file: MANIFEST-000009 size: 3005 Bytes 2024-02-07T12:48:22.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000057.sst 2024-02-07T12:48:22.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000055.log size: 2153384 ; 2024-02-07T12:48:22.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.error_if_exists: 0 2024-02-07T12:48:22.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.create_if_missing: 0 2024-02-07T12:48:22.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.paranoid_checks: 1 2024-02-07T12:48:22.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.flush_verify_memtable_count: 1 2024-02-07T12:48:22.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-02-07T12:48:22.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-02-07T12:48:22.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.env: 0x558bb9a9eae0 2024-02-07T12:48:22.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.fs: PosixFileSystem 2024-02-07T12:48:22.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.info_log: 0x558bbb01f8c0 2024-02-07T12:48:22.846 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.max_file_opening_threads: 16 2024-02-07T12:48:22.847 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.statistics: (nil) 2024-02-07T12:48:22.847 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.use_fsync: 0 2024-02-07T12:48:22.847 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.max_log_file_size: 0 2024-02-07T12:48:22.848 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-02-07T12:48:22.848 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.log_file_time_to_roll: 0 2024-02-07T12:48:22.848 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.keep_log_file_num: 1000 2024-02-07T12:48:22.848 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.recycle_log_file_num: 0 2024-02-07T12:48:22.848 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.allow_fallocate: 1 2024-02-07T12:48:22.848 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.allow_mmap_reads: 0 2024-02-07T12:48:22.848 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.allow_mmap_writes: 0 2024-02-07T12:48:22.848 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.use_direct_reads: 0 2024-02-07T12:48:22.848 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-02-07T12:48:22.848 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.create_missing_column_families: 0 2024-02-07T12:48:22.848 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.db_log_dir: 2024-02-07T12:48:22.848 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.wal_dir: 2024-02-07T12:48:22.848 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.table_cache_numshardbits: 6 2024-02-07T12:48:22.848 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.WAL_ttl_seconds: 0 2024-02-07T12:48:22.849 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.WAL_size_limit_MB: 0 2024-02-07T12:48:22.849 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-02-07T12:48:22.849 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-02-07T12:48:22.849 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.is_fd_close_on_exec: 1 2024-02-07T12:48:22.849 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.advise_random_on_open: 1 2024-02-07T12:48:22.849 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.db_write_buffer_size: 0 2024-02-07T12:48:22.849 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.write_buffer_manager: 0x558bbaff2dc0 2024-02-07T12:48:22.849 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-02-07T12:48:22.849 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-02-07T12:48:22.849 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.use_adaptive_mutex: 0 2024-02-07T12:48:22.849 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.rate_limiter: (nil) 2024-02-07T12:48:22.849 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-02-07T12:48:22.849 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.wal_recovery_mode: 2 2024-02-07T12:48:22.849 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.enable_thread_tracking: 0 2024-02-07T12:48:22.849 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.enable_pipelined_write: 0 2024-02-07T12:48:22.850 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.unordered_write: 0 2024-02-07T12:48:22.850 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-02-07T12:48:22.850 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-02-07T12:48:22.850 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-02-07T12:48:22.850 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-02-07T12:48:22.850 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.row_cache: None 2024-02-07T12:48:22.850 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.wal_filter: None 2024-02-07T12:48:22.850 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-02-07T12:48:22.850 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.allow_ingest_behind: 0 2024-02-07T12:48:22.850 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.two_write_queues: 0 2024-02-07T12:48:22.850 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.manual_wal_flush: 0 2024-02-07T12:48:22.850 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.wal_compression: 0 2024-02-07T12:48:22.850 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.atomic_flush: 0 2024-02-07T12:48:22.850 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-02-07T12:48:22.850 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.persist_stats_to_disk: 0 2024-02-07T12:48:22.851 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.write_dbid_to_manifest: 0 2024-02-07T12:48:22.851 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.log_readahead_size: 0 2024-02-07T12:48:22.851 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-02-07T12:48:22.851 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.best_efforts_recovery: 0 2024-02-07T12:48:22.851 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-02-07T12:48:22.851 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-02-07T12:48:22.851 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.allow_data_in_errors: 0 2024-02-07T12:48:22.851 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.db_host_id: __hostname__ 2024-02-07T12:48:22.851 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.enforce_single_del_contracts: true 2024-02-07T12:48:22.851 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.max_background_jobs: 2 2024-02-07T12:48:22.851 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.max_background_compactions: -1 2024-02-07T12:48:22.851 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.max_subcompactions: 1 2024-02-07T12:48:22.851 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-02-07T12:48:22.851 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-02-07T12:48:22.851 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.delayed_write_rate : 16777216 2024-02-07T12:48:22.852 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.max_total_wal_size: 0 2024-02-07T12:48:22.852 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-02-07T12:48:22.852 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.stats_dump_period_sec: 600 2024-02-07T12:48:22.852 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.stats_persist_period_sec: 600 2024-02-07T12:48:22.852 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-02-07T12:48:22.852 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.max_open_files: -1 2024-02-07T12:48:22.852 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.bytes_per_sync: 0 2024-02-07T12:48:22.852 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.wal_bytes_per_sync: 0 2024-02-07T12:48:22.852 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.strict_bytes_per_sync: 0 2024-02-07T12:48:22.852 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.compaction_readahead_size: 0 2024-02-07T12:48:22.852 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.max_background_flushes: -1 2024-02-07T12:48:22.852 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Compression algorithms supported: 2024-02-07T12:48:22.852 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-02-07T12:48:22.852 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: kZSTD supported: 0 2024-02-07T12:48:22.852 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: kXpressCompression supported: 0 2024-02-07T12:48:22.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: kLZ4HCCompression supported: 1 2024-02-07T12:48:22.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: kZlibCompression supported: 1 2024-02-07T12:48:22.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: kSnappyCompression supported: 1 2024-02-07T12:48:22.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: kLZ4Compression supported: 1 2024-02-07T12:48:22.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: kBZip2Compression supported: 0 2024-02-07T12:48:22.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Fast CRC32 supported: Supported on x86 2024-02-07T12:48:22.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: DMutex implementation: pthread_mutex_t 2024-02-07T12:48:22.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-02-07T12:48:22.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-02-07T12:48:22.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-02-07T12:48:22.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.merge_operator: 2024-02-07T12:48:22.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.compaction_filter: None 2024-02-07T12:48:22.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.compaction_filter_factory: None 2024-02-07T12:48:22.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.sst_partitioner_factory: None 2024-02-07T12:48:22.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.memtable_factory: SkipListFactory 2024-02-07T12:48:22.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.table_factory: BlockBasedTable 2024-02-07T12:48:22.854 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558bbb01f9c0) 2024-02-07T12:48:22.854 INFO:journalctl@ceph.mon.c.smithi019.stdout: cache_index_and_filter_blocks: 1 2024-02-07T12:48:22.854 INFO:journalctl@ceph.mon.c.smithi019.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-02-07T12:48:22.854 INFO:journalctl@ceph.mon.c.smithi019.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-02-07T12:48:22.854 INFO:journalctl@ceph.mon.c.smithi019.stdout: pin_top_level_index_and_filter: 1 2024-02-07T12:48:22.854 INFO:journalctl@ceph.mon.c.smithi019.stdout: index_type: 0 2024-02-07T12:48:22.854 INFO:journalctl@ceph.mon.c.smithi019.stdout: data_block_index_type: 0 2024-02-07T12:48:22.854 INFO:journalctl@ceph.mon.c.smithi019.stdout: index_shortening: 1 2024-02-07T12:48:22.854 INFO:journalctl@ceph.mon.c.smithi019.stdout: data_block_hash_table_util_ratio: 0.750000 2024-02-07T12:48:22.854 INFO:journalctl@ceph.mon.c.smithi019.stdout: checksum: 4 2024-02-07T12:48:22.854 INFO:journalctl@ceph.mon.c.smithi019.stdout: no_block_cache: 0 2024-02-07T12:48:22.854 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_cache: 0x558bbb023090 2024-02-07T12:48:22.854 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_cache_name: BinnedLRUCache 2024-02-07T12:48:22.854 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_cache_options: 2024-02-07T12:48:22.855 INFO:journalctl@ceph.mon.c.smithi019.stdout: capacity : 536870912 2024-02-07T12:48:22.855 INFO:journalctl@ceph.mon.c.smithi019.stdout: num_shard_bits : 4 2024-02-07T12:48:22.855 INFO:journalctl@ceph.mon.c.smithi019.stdout: strict_capacity_limit : 0 2024-02-07T12:48:22.855 INFO:journalctl@ceph.mon.c.smithi019.stdout: high_pri_pool_ratio: 0.000 2024-02-07T12:48:22.855 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_cache_compressed: (nil) 2024-02-07T12:48:22.855 INFO:journalctl@ceph.mon.c.smithi019.stdout: persistent_cache: (nil) 2024-02-07T12:48:22.855 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_size: 4096 2024-02-07T12:48:22.855 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_size_deviation: 10 2024-02-07T12:48:22.855 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_restart_interval: 16 2024-02-07T12:48:22.855 INFO:journalctl@ceph.mon.c.smithi019.stdout: index_block_restart_interval: 1 2024-02-07T12:48:22.855 INFO:journalctl@ceph.mon.c.smithi019.stdout: metadata_block_size: 4096 2024-02-07T12:48:22.855 INFO:journalctl@ceph.mon.c.smithi019.stdout: partition_filters: 0 2024-02-07T12:48:22.855 INFO:journalctl@ceph.mon.c.smithi019.stdout: use_delta_encoding: 1 2024-02-07T12:48:22.855 INFO:journalctl@ceph.mon.c.smithi019.stdout: filter_policy: bloomfilter 2024-02-07T12:48:22.855 INFO:journalctl@ceph.mon.c.smithi019.stdout: whole_key_filtering: 1 2024-02-07T12:48:22.855 INFO:journalctl@ceph.mon.c.smithi019.stdout: verify_compression: 0 2024-02-07T12:48:22.856 INFO:journalctl@ceph.mon.c.smithi019.stdout: read_amp_bytes_per_bit: 0 2024-02-07T12:48:22.856 INFO:journalctl@ceph.mon.c.smithi019.stdout: format_version: 5 2024-02-07T12:48:22.856 INFO:journalctl@ceph.mon.c.smithi019.stdout: enable_index_compression: 1 2024-02-07T12:48:22.856 INFO:journalctl@ceph.mon.c.smithi019.stdout: block_align: 0 2024-02-07T12:48:22.856 INFO:journalctl@ceph.mon.c.smithi019.stdout: max_auto_readahead_size: 262144 2024-02-07T12:48:22.856 INFO:journalctl@ceph.mon.c.smithi019.stdout: prepopulate_block_cache: 0 2024-02-07T12:48:22.856 INFO:journalctl@ceph.mon.c.smithi019.stdout: initial_auto_readahead_size: 8192 2024-02-07T12:48:22.856 INFO:journalctl@ceph.mon.c.smithi019.stdout: num_file_reads_for_auto_readahead: 2 2024-02-07T12:48:22.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.write_buffer_size: 33554432 2024-02-07T12:48:22.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.max_write_buffer_number: 2 2024-02-07T12:48:22.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.compression: NoCompression 2024-02-07T12:48:22.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.bottommost_compression: Disabled 2024-02-07T12:48:22.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.prefix_extractor: nullptr 2024-02-07T12:48:22.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-02-07T12:48:22.856 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.num_levels: 7 2024-02-07T12:48:22.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-02-07T12:48:22.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-02-07T12:48:22.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-02-07T12:48:22.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-02-07T12:48:22.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-02-07T12:48:22.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-02-07T12:48:22.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-02-07T12:48:22.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-02-07T12:48:22.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-02-07T12:48:22.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-02-07T12:48:22.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-02-07T12:48:22.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-02-07T12:48:22.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.compression_opts.window_bits: -14 2024-02-07T12:48:22.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.compression_opts.level: 32767 2024-02-07T12:48:22.857 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.compression_opts.strategy: 0 2024-02-07T12:48:22.858 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-02-07T12:48:22.858 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-02-07T12:48:22.858 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-02-07T12:48:22.858 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-02-07T12:48:22.858 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.compression_opts.enabled: false 2024-02-07T12:48:22.858 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-02-07T12:48:22.858 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-02-07T12:48:22.858 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-02-07T12:48:22.858 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-02-07T12:48:22.858 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.target_file_size_base: 67108864 2024-02-07T12:48:22.858 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.target_file_size_multiplier: 1 2024-02-07T12:48:22.858 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-02-07T12:48:22.858 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-02-07T12:48:22.858 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-02-07T12:48:22.858 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-02-07T12:48:22.858 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-02-07T12:48:22.859 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-02-07T12:48:22.859 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-02-07T12:48:22.859 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-02-07T12:48:22.859 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-02-07T12:48:22.859 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-02-07T12:48:22.859 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-02-07T12:48:22.859 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-02-07T12:48:22.859 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-02-07T12:48:22.859 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.arena_block_size: 1048576 2024-02-07T12:48:22.859 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-02-07T12:48:22.859 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-02-07T12:48:22.859 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.disable_auto_compactions: 0 2024-02-07T12:48:22.859 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-02-07T12:48:22.859 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-02-07T12:48:22.859 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-02-07T12:48:22.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-02-07T12:48:22.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-02-07T12:48:22.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-02-07T12:48:22.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-02-07T12:48:22.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-02-07T12:48:22.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-02-07T12:48:22.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-02-07T12:48:22.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.table_properties_collectors: 2024-02-07T12:48:22.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.inplace_update_support: 0 2024-02-07T12:48:22.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.inplace_update_num_locks: 10000 2024-02-07T12:48:22.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-02-07T12:48:22.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-02-07T12:48:22.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.memtable_huge_page_size: 0 2024-02-07T12:48:22.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.bloom_locality: 0 2024-02-07T12:48:22.860 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.max_successive_merges: 0 2024-02-07T12:48:22.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.optimize_filters_for_hits: 0 2024-02-07T12:48:22.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.paranoid_file_checks: 0 2024-02-07T12:48:22.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.force_consistency_checks: 1 2024-02-07T12:48:22.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.report_bg_io_stats: 0 2024-02-07T12:48:22.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.ttl: 2592000 2024-02-07T12:48:22.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.periodic_compaction_seconds: 0 2024-02-07T12:48:22.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-02-07T12:48:22.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-02-07T12:48:22.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.enable_blob_files: false 2024-02-07T12:48:22.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.min_blob_size: 0 2024-02-07T12:48:22.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.blob_file_size: 268435456 2024-02-07T12:48:22.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.blob_compression_type: NoCompression 2024-02-07T12:48:22.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.enable_blob_garbage_collection: false 2024-02-07T12:48:22.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-02-07T12:48:22.861 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-02-07T12:48:22.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-02-07T12:48:22.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.blob_file_starting_level: 0 2024-02-07T12:48:22.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-02-07T12:48:22.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 57.sst 2024-02-07T12:48:22.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-02-07T12:48:22.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 59, last_sequence is 31820, log_number is 55,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-02-07T12:48:22.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 55 2024-02-07T12:48:22.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 14ff0c38-ce13-4ecb-a9dc-f7eb8f30f00b 2024-02-07T12:48:22.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707310102589859, "job": 1, "event": "recovery_started", "wal_files": [55]} 2024-02-07T12:48:22.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #55 mode 2 2024-02-07T12:48:22.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707310102595582, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 60, "file_size": 1352576, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 31825, "largest_seqno": 32281, "table_properties": {"data_size": 1349539, "index_size": 1447, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 5262, "raw_average_key_size": 24, "raw_value_size": 1344748, "raw_average_value_size": 6196, "num_data_blocks": 65, "num_entries": 217, "num_filter_entries": 217, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1707310102, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "14ff0c38-ce13-4ecb-a9dc-f7eb8f30f00b", "db_session_id": "PBA6UCWZMIUGOTNPMDJW", "orig_file_number": 60, "seqno_to_time_mapping": "N/A"}} 2024-02-07T12:48:22.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707310102595679, "job": 1, "event": "recovery_finished"} 2024-02-07T12:48:22.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: [db/version_set.cc:5047] Creating manifest 62 2024-02-07T12:48:22.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-02-07T12:48:22.862 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000055.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-02-07T12:48:22.863 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x558bbb114000 2024-02-07T12:48:22.863 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: DB pointer 0x558bbb0fe000 2024-02-07T12:48:22.863 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-02-07T12:48:22.863 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-02-07T12:48:22.863 INFO:journalctl@ceph.mon.c.smithi019.stdout: ** DB Stats ** 2024-02-07T12:48:22.863 INFO:journalctl@ceph.mon.c.smithi019.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-02-07T12:48:22.863 INFO:journalctl@ceph.mon.c.smithi019.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-02-07T12:48:22.863 INFO:journalctl@ceph.mon.c.smithi019.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-02-07T12:48:22.863 INFO:journalctl@ceph.mon.c.smithi019.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-07T12:48:22.863 INFO:journalctl@ceph.mon.c.smithi019.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-02-07T12:48:22.863 INFO:journalctl@ceph.mon.c.smithi019.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-02-07T12:48:22.863 INFO:journalctl@ceph.mon.c.smithi019.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-07T12:48:22.863 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2024-02-07T12:48:22.863 INFO:journalctl@ceph.mon.c.smithi019.stdout: ** Compaction Stats [default] ** 2024-02-07T12:48:22.863 INFO:journalctl@ceph.mon.c.smithi019.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-02-07T12:48:22.864 INFO:journalctl@ceph.mon.c.smithi019.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-02-07T12:48:22.864 INFO:journalctl@ceph.mon.c.smithi019.stdout: L0 1/0 1.29 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 461.5 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-02-07T12:48:22.864 INFO:journalctl@ceph.mon.c.smithi019.stdout: L6 1/0 10.29 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-02-07T12:48:22.864 INFO:journalctl@ceph.mon.c.smithi019.stdout: Sum 2/0 11.58 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 461.5 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-02-07T12:48:22.864 INFO:journalctl@ceph.mon.c.smithi019.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 461.5 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-02-07T12:48:22.864 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2024-02-07T12:48:22.864 INFO:journalctl@ceph.mon.c.smithi019.stdout: ** Compaction Stats [default] ** 2024-02-07T12:48:22.864 INFO:journalctl@ceph.mon.c.smithi019.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-02-07T12:48:22.864 INFO:journalctl@ceph.mon.c.smithi019.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-07T12:48:22.864 INFO:journalctl@ceph.mon.c.smithi019.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 461.5 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-02-07T12:48:22.864 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2024-02-07T12:48:22.864 INFO:journalctl@ceph.mon.c.smithi019.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-02-07T12:48:22.864 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2024-02-07T12:48:22.864 INFO:journalctl@ceph.mon.c.smithi019.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-02-07T12:48:22.864 INFO:journalctl@ceph.mon.c.smithi019.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-02-07T12:48:22.865 INFO:journalctl@ceph.mon.c.smithi019.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-02-07T12:48:22.865 INFO:journalctl@ceph.mon.c.smithi019.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-02-07T12:48:22.865 INFO:journalctl@ceph.mon.c.smithi019.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-02-07T12:48:22.865 INFO:journalctl@ceph.mon.c.smithi019.stdout: AddFile(Keys): cumulative 0, interval 0 2024-02-07T12:48:22.865 INFO:journalctl@ceph.mon.c.smithi019.stdout: Cumulative compaction: 0.00 GB write, 161.18 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:48:22.865 INFO:journalctl@ceph.mon.c.smithi019.stdout: Interval compaction: 0.00 GB write, 161.18 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:48:22.865 INFO:journalctl@ceph.mon.c.smithi019.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-02-07T12:48:22.865 INFO:journalctl@ceph.mon.c.smithi019.stdout: Block cache BinnedLRUCache@0x558bbb023090#2 capacity: 512.00 MB usage: 2.23 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.5e-05 secs_since: 0 2024-02-07T12:48:22.865 INFO:journalctl@ceph.mon.c.smithi019.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.66 KB,0.00012517%) IndexBlock(1,1.58 KB,0.000301003%) Misc(1,0.00 KB,0%) 2024-02-07T12:48:22.865 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2024-02-07T12:48:22.865 INFO:journalctl@ceph.mon.c.smithi019.stdout: ** File Read Latency Histogram By Level [default] ** 2024-02-07T12:48:22.865 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: starting mon.c rank 1 at public addrs [v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0] at bind addrs [v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:48:22.865 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: mon.c@-1(???) e3 preinit fsid ced2783c-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:48:22.865 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: mon.c@-1(???).mds e1 new map 2024-02-07T12:48:22.865 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: mon.c@-1(???).mds e1 print_map 2024-02-07T12:48:22.866 INFO:journalctl@ceph.mon.c.smithi019.stdout: e1 2024-02-07T12:48:22.866 INFO:journalctl@ceph.mon.c.smithi019.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-02-07T12:48:22.866 INFO:journalctl@ceph.mon.c.smithi019.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-02-07T12:48:22.866 INFO:journalctl@ceph.mon.c.smithi019.stdout: legacy client fscid: -1 2024-02-07T12:48:22.866 INFO:journalctl@ceph.mon.c.smithi019.stdout: 2024-02-07T12:48:22.866 INFO:journalctl@ceph.mon.c.smithi019.stdout: No filesystems configured 2024-02-07T12:48:22.866 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: mon.c@-1(???).osd e105 crush map has features 3314933000854323200, adjusting msgr requires 2024-02-07T12:48:22.866 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: mon.c@-1(???).osd e105 crush map has features 432629239337189376, adjusting msgr requires 2024-02-07T12:48:22.866 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: mon.c@-1(???).osd e105 crush map has features 432629239337189376, adjusting msgr requires 2024-02-07T12:48:22.866 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: mon.c@-1(???).osd e105 crush map has features 432629239337189376, adjusting msgr requires 2024-02-07T12:48:22.866 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 ceph-mon[88137]: mon.c@-1(???).paxosservice(auth 1..27) refresh upgraded, format 0 -> 3 2024-02-07T12:48:22.866 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 bash[88119]: 6214003f6e08b6bc4e7871cf9b636365217102d9b7431aadb7f35f80718c4d9d 2024-02-07T12:48:22.866 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:22 smithi019 systemd[1]: Started Ceph mon.c for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:48:24.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:23 smithi172 ceph-mon[67636]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-02-07T12:48:24.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:23 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:48:24.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:23 smithi019 ceph-mon[82918]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-02-07T12:48:24.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:23 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:48:24.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:48:23] "GET /metrics HTTP/1.1" 200 38004 "" "Prometheus/2.43.0" 2024-02-07T12:48:24.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:23 smithi019 ceph-mon[88137]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-02-07T12:48:24.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:23 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:48:28.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:27 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:48:27.604+0000 7fb74ced9700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-02-07T12:48:30.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 996 B/s rd, 0 op/s 2024-02-07T12:48:30.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: mon.c calling monitor election 2024-02-07T12:48:30.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:48:30.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: mon.b calling monitor election 2024-02-07T12:48:30.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:48:30.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:48:30.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 996 B/s rd, 0 op/s 2024-02-07T12:48:30.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:48:30.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:48:30.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: monmap e4: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-02-07T12:48:30.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: fsmap 2024-02-07T12:48:30.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: osdmap e105: 8 total, 8 up, 8 in 2024-02-07T12:48:30.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: mgrmap e49: y(active, since 23s), standbys: x 2024-02-07T12:48:30.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-02-07T12:48:30.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-02-07T12:48:30.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-02-07T12:48:30.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: daemon iscsi.foo.smithi019.svjimj on smithi019 is in unknown state 2024-02-07T12:48:30.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: mon.a calling monitor election 2024-02-07T12:48:30.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: mon.c calling monitor election 2024-02-07T12:48:30.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-02-07T12:48:30.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: monmap e4: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-02-07T12:48:30.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: fsmap 2024-02-07T12:48:30.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: osdmap e105: 8 total, 8 up, 8 in 2024-02-07T12:48:30.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: mgrmap e49: y(active, since 23s), standbys: x 2024-02-07T12:48:30.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-02-07T12:48:30.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-02-07T12:48:30.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-02-07T12:48:30.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: daemon iscsi.foo.smithi019.svjimj on smithi019 is in unknown state 2024-02-07T12:48:30.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='' 2024-02-07T12:48:30.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:30.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:48:30.099 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 996 B/s rd, 0 op/s 2024-02-07T12:48:30.099 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: mon.c calling monitor election 2024-02-07T12:48:30.099 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:48:30.099 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: mon.b calling monitor election 2024-02-07T12:48:30.099 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:48:30.099 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:48:30.100 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 996 B/s rd, 0 op/s 2024-02-07T12:48:30.100 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:48:30.100 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:48:30.100 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: monmap e4: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-02-07T12:48:30.100 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: fsmap 2024-02-07T12:48:30.100 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: osdmap e105: 8 total, 8 up, 8 in 2024-02-07T12:48:30.100 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: mgrmap e49: y(active, since 23s), standbys: x 2024-02-07T12:48:30.100 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-02-07T12:48:30.101 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-02-07T12:48:30.101 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-02-07T12:48:30.101 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: daemon iscsi.foo.smithi019.svjimj on smithi019 is in unknown state 2024-02-07T12:48:30.101 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: mon.a calling monitor election 2024-02-07T12:48:30.101 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: mon.c calling monitor election 2024-02-07T12:48:30.101 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-02-07T12:48:30.101 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: monmap e4: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-02-07T12:48:30.101 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: fsmap 2024-02-07T12:48:30.101 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: osdmap e105: 8 total, 8 up, 8 in 2024-02-07T12:48:30.101 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: mgrmap e49: y(active, since 23s), standbys: x 2024-02-07T12:48:30.102 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-02-07T12:48:30.102 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-02-07T12:48:30.102 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-02-07T12:48:30.102 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: daemon iscsi.foo.smithi019.svjimj on smithi019 is in unknown state 2024-02-07T12:48:30.102 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='' 2024-02-07T12:48:30.102 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:30.102 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:29 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:48:30.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 996 B/s rd, 0 op/s 2024-02-07T12:48:30.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: mon.c calling monitor election 2024-02-07T12:48:30.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-02-07T12:48:30.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: mon.b calling monitor election 2024-02-07T12:48:30.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-02-07T12:48:30.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-02-07T12:48:30.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 996 B/s rd, 0 op/s 2024-02-07T12:48:30.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:48:30.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:48:30.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: monmap e4: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-02-07T12:48:30.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: fsmap 2024-02-07T12:48:30.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: osdmap e105: 8 total, 8 up, 8 in 2024-02-07T12:48:30.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: mgrmap e49: y(active, since 23s), standbys: x 2024-02-07T12:48:30.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-02-07T12:48:30.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-02-07T12:48:30.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-02-07T12:48:30.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: daemon iscsi.foo.smithi019.svjimj on smithi019 is in unknown state 2024-02-07T12:48:30.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: mon.a calling monitor election 2024-02-07T12:48:30.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: mon.c calling monitor election 2024-02-07T12:48:30.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-02-07T12:48:30.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: monmap e4: 3 mons at {a=[v2:172.21.15.19:3300/0,v1:172.21.15.19:6789/0],b=[v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0],c=[v2:172.21.15.19:3301/0,v1:172.21.15.19:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-02-07T12:48:30.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: fsmap 2024-02-07T12:48:30.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: osdmap e105: 8 total, 8 up, 8 in 2024-02-07T12:48:30.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: mgrmap e49: y(active, since 23s), standbys: x 2024-02-07T12:48:30.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-02-07T12:48:30.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-02-07T12:48:30.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-02-07T12:48:30.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: daemon iscsi.foo.smithi019.svjimj on smithi019 is in unknown state 2024-02-07T12:48:30.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='' 2024-02-07T12:48:30.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:30.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:29 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:48:31.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:30 smithi019 ceph-mon[88137]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:48:31.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:30 smithi019 ceph-mon[82918]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:48:31.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:30 smithi172 ceph-mon[67636]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:48:33.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:32 smithi019 ceph-mon[82918]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:48:33.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:32 smithi019 ceph-mon[88137]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:48:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:32 smithi172 ceph-mon[67636]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:48:34.202 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:48:33] "GET /metrics HTTP/1.1" 200 38004 "" "Prometheus/2.43.0" 2024-02-07T12:48:34.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:34.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:34.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:34 smithi019 ceph-mon[88137]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:48:34.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:34.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:34.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:34 smithi019 ceph-mon[82918]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:48:34.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:34.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:34.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:34 smithi172 ceph-mon[67636]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:48:35.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:35 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:35.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:35 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:35.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:35 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:48:35.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:35 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:35.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:35 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:35.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:35 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:48:35.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:35 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:35.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:35 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:35.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:35 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:48:37.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:36 smithi019 ceph-mon[82918]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:48:37.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:36 smithi019 ceph-mon[88137]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:48:37.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:36 smithi172 ceph-mon[67636]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:48:38.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:37 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:48:38.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:38.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:38.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:38.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:48:38.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:38.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:48:38.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:48:38.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:38.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:37 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:48:38.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:38.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:38.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:38.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:48:38.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:38.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:48:38.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:48:38.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:38.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:37 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:48:38.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:38.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:38.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:38.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:48:38.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:38.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:48:38.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:48:38.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:39.091 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:38 smithi019 ceph-mon[88137]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:48:39.092 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:38 smithi019 ceph-mon[88137]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 980 B/s rd, 0 op/s 2024-02-07T12:48:39.092 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:38 smithi019 ceph-mon[88137]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:48:39.092 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:38 smithi019 ceph-mon[88137]: Reconfiguring mon.a (monmap changed)... 2024-02-07T12:48:39.092 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:38 smithi019 ceph-mon[88137]: Reconfiguring daemon mon.a on smithi019 2024-02-07T12:48:39.092 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:38 smithi019 ceph-mon[82918]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:48:39.092 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:38 smithi019 ceph-mon[82918]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 980 B/s rd, 0 op/s 2024-02-07T12:48:39.092 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:38 smithi019 ceph-mon[82918]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:48:39.092 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:38 smithi019 ceph-mon[82918]: Reconfiguring mon.a (monmap changed)... 2024-02-07T12:48:39.092 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:38 smithi019 ceph-mon[82918]: Reconfiguring daemon mon.a on smithi019 2024-02-07T12:48:39.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:38 smithi172 ceph-mon[67636]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:48:39.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:38 smithi172 ceph-mon[67636]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 980 B/s rd, 0 op/s 2024-02-07T12:48:39.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:38 smithi172 ceph-mon[67636]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:48:39.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:38 smithi172 ceph-mon[67636]: Reconfiguring mon.a (monmap changed)... 2024-02-07T12:48:39.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:38 smithi172 ceph-mon[67636]: Reconfiguring daemon mon.a on smithi019 2024-02-07T12:48:40.680 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:40.681 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:40.681 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:40 smithi019 ceph-mon[82918]: Reconfiguring mgr.y (monmap changed)... 2024-02-07T12:48:40.681 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:40 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:48:40.681 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:48:40.681 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:40 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:48:40.681 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:40 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:40.681 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:40 smithi019 ceph-mon[82918]: Reconfiguring daemon mgr.y on smithi019 2024-02-07T12:48:40.681 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:40 smithi019 ceph-mon[82918]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 606 B/s rd, 0 op/s 2024-02-07T12:48:40.681 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:40.681 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:40.682 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:40 smithi019 ceph-mon[88137]: Reconfiguring mgr.y (monmap changed)... 2024-02-07T12:48:40.682 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:40 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:48:40.682 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:48:40.682 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:40 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:48:40.682 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:40 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:40.682 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:40 smithi019 ceph-mon[88137]: Reconfiguring daemon mgr.y on smithi019 2024-02-07T12:48:40.682 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:40 smithi019 ceph-mon[88137]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 606 B/s rd, 0 op/s 2024-02-07T12:48:40.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:40.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:40.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:40 smithi172 ceph-mon[67636]: Reconfiguring mgr.y (monmap changed)... 2024-02-07T12:48:40.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:40 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:48:40.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:48:40.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:40 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:48:40.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:40 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:40.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:40 smithi172 ceph-mon[67636]: Reconfiguring daemon mgr.y on smithi019 2024-02-07T12:48:40.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:40 smithi172 ceph-mon[67636]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 606 B/s rd, 0 op/s 2024-02-07T12:48:42.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:42 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:42.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:42 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:42.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:42 smithi019 ceph-mon[88137]: Reconfiguring mon.c (monmap changed)... 2024-02-07T12:48:42.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:42 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:48:42.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:42 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:48:42.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:42 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:42.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:42 smithi019 ceph-mon[88137]: Reconfiguring daemon mon.c on smithi019 2024-02-07T12:48:42.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:42 smithi019 ceph-mon[88137]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:48:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:42 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:42 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:42 smithi019 ceph-mon[82918]: Reconfiguring mon.c (monmap changed)... 2024-02-07T12:48:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:42 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:48:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:42 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:48:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:42 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:42 smithi019 ceph-mon[82918]: Reconfiguring daemon mon.c on smithi019 2024-02-07T12:48:42.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:42 smithi019 ceph-mon[82918]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:48:42.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:42 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:42.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:42 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:42.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:42 smithi172 ceph-mon[67636]: Reconfiguring mon.c (monmap changed)... 2024-02-07T12:48:42.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:42 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:48:42.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:42 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:48:42.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:42 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:42.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:42 smithi172 ceph-mon[67636]: Reconfiguring daemon mon.c on smithi019 2024-02-07T12:48:42.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:42 smithi172 ceph-mon[67636]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:48:44.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:48:43] "GET /metrics HTTP/1.1" 200 38020 "" "Prometheus/2.43.0" 2024-02-07T12:48:44.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:43 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:44.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:43 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:44.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:43 smithi019 ceph-mon[88137]: Reconfiguring osd.0 (monmap changed)... 2024-02-07T12:48:44.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:43 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-02-07T12:48:44.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:43 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:44.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:43 smithi019 ceph-mon[88137]: Reconfiguring daemon osd.0 on smithi019 2024-02-07T12:48:44.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:43 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:44.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:43 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:44.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:43 smithi019 ceph-mon[82918]: Reconfiguring osd.0 (monmap changed)... 2024-02-07T12:48:44.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:43 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-02-07T12:48:44.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:43 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:44.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:43 smithi019 ceph-mon[82918]: Reconfiguring daemon osd.0 on smithi019 2024-02-07T12:48:44.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:43 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:44.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:43 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:44.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:43 smithi172 ceph-mon[67636]: Reconfiguring osd.0 (monmap changed)... 2024-02-07T12:48:44.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:43 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-02-07T12:48:44.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:43 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:44.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:43 smithi172 ceph-mon[67636]: Reconfiguring daemon osd.0 on smithi019 2024-02-07T12:48:45.111 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:44 smithi019 ceph-mon[88137]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:48:45.111 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:44 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:45.111 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:44 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:45.111 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:44 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-02-07T12:48:45.111 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:44 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:45.111 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:44 smithi019 ceph-mon[82918]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:48:45.111 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:44 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:45.111 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:44 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:45.112 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:44 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-02-07T12:48:45.112 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:44 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:45.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:44 smithi172 ceph-mon[67636]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:48:45.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:44 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:45.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:44 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:45.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:44 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-02-07T12:48:45.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:44 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:46.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:45 smithi019 ceph-mon[88137]: Reconfiguring osd.1 (monmap changed)... 2024-02-07T12:48:46.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:45 smithi019 ceph-mon[88137]: Reconfiguring daemon osd.1 on smithi019 2024-02-07T12:48:46.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:45 smithi019 ceph-mon[82918]: Reconfiguring osd.1 (monmap changed)... 2024-02-07T12:48:46.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:45 smithi019 ceph-mon[82918]: Reconfiguring daemon osd.1 on smithi019 2024-02-07T12:48:46.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:45 smithi172 ceph-mon[67636]: Reconfiguring osd.1 (monmap changed)... 2024-02-07T12:48:46.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:45 smithi172 ceph-mon[67636]: Reconfiguring daemon osd.1 on smithi019 2024-02-07T12:48:47.118 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:46 smithi019 ceph-mon[82918]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 606 B/s rd, 0 op/s 2024-02-07T12:48:47.118 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:46 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:47.118 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:46 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:47.118 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:46 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-02-07T12:48:47.118 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:46 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:47.118 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:46 smithi019 ceph-mon[88137]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 606 B/s rd, 0 op/s 2024-02-07T12:48:47.118 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:46 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:47.119 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:46 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:47.119 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:46 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-02-07T12:48:47.119 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:46 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:46 smithi172 ceph-mon[67636]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 606 B/s rd, 0 op/s 2024-02-07T12:48:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:46 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:46 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:46 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-02-07T12:48:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:46 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:48.000 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:47 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:48:48.001 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:47 smithi019 ceph-mon[82918]: Reconfiguring osd.2 (monmap changed)... 2024-02-07T12:48:48.001 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:47 smithi019 ceph-mon[82918]: Reconfiguring daemon osd.2 on smithi019 2024-02-07T12:48:48.001 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:47 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:48:48.001 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:47 smithi019 ceph-mon[88137]: Reconfiguring osd.2 (monmap changed)... 2024-02-07T12:48:48.001 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:47 smithi019 ceph-mon[88137]: Reconfiguring daemon osd.2 on smithi019 2024-02-07T12:48:48.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:47 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:48:48.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:47 smithi172 ceph-mon[67636]: Reconfiguring osd.2 (monmap changed)... 2024-02-07T12:48:48.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:47 smithi172 ceph-mon[67636]: Reconfiguring daemon osd.2 on smithi019 2024-02-07T12:48:49.054 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:48 smithi019 ceph-mon[82918]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:48:49.054 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:49.054 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:49.055 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-02-07T12:48:49.055 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:49.055 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:48 smithi019 ceph-mon[88137]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:48:49.055 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:49.055 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:49.055 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-02-07T12:48:49.055 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:49.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:48 smithi172 ceph-mon[67636]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:48:49.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:49.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:49.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-02-07T12:48:49.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:50.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:49 smithi019 ceph-mon[82918]: Reconfiguring osd.3 (monmap changed)... 2024-02-07T12:48:50.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:49 smithi019 ceph-mon[82918]: Reconfiguring daemon osd.3 on smithi019 2024-02-07T12:48:50.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:49 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:50.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:49 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:50.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:49 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi019.ciualv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:48:50.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:49 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi019.ciualv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:48:50.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:49 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:50.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:49 smithi019 ceph-mon[88137]: Reconfiguring osd.3 (monmap changed)... 2024-02-07T12:48:50.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:49 smithi019 ceph-mon[88137]: Reconfiguring daemon osd.3 on smithi019 2024-02-07T12:48:50.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:49 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:50.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:49 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:50.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:49 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi019.ciualv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:48:50.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:49 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi019.ciualv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:48:50.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:49 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:50.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:49 smithi172 ceph-mon[67636]: Reconfiguring osd.3 (monmap changed)... 2024-02-07T12:48:50.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:49 smithi172 ceph-mon[67636]: Reconfiguring daemon osd.3 on smithi019 2024-02-07T12:48:50.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:49 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:50.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:49 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:50.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:49 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi019.ciualv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:48:50.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:49 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi019.ciualv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:48:50.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:49 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:51.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:50 smithi019 ceph-mon[88137]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:48:51.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:50 smithi019 ceph-mon[88137]: Reconfiguring rgw.foo.smithi019.ciualv (monmap changed)... 2024-02-07T12:48:51.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:50 smithi019 ceph-mon[88137]: Reconfiguring daemon rgw.foo.smithi019.ciualv on smithi019 2024-02-07T12:48:51.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:50 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:48:51.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:50 smithi019 ceph-mon[82918]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:48:51.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:50 smithi019 ceph-mon[82918]: Reconfiguring rgw.foo.smithi019.ciualv (monmap changed)... 2024-02-07T12:48:51.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:50 smithi019 ceph-mon[82918]: Reconfiguring daemon rgw.foo.smithi019.ciualv on smithi019 2024-02-07T12:48:51.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:50 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:48:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:50 smithi172 ceph-mon[67636]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:48:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:50 smithi172 ceph-mon[67636]: Reconfiguring rgw.foo.smithi019.ciualv (monmap changed)... 2024-02-07T12:48:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:50 smithi172 ceph-mon[67636]: Reconfiguring daemon rgw.foo.smithi019.ciualv on smithi019 2024-02-07T12:48:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:50 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:48:51.866 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:48:52.553 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:52 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:52.553 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:52 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:52.553 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:52 smithi019 ceph-mon[82918]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:48:52.554 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:52 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:48:52.554 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:52 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:48:52.554 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:52 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:52.554 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:52 smithi019 ceph-mon[82918]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:48:52.554 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:52 smithi019 ceph-mon[82918]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:48:52.554 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:52 smithi019 ceph-mon[82918]: from='client.44166 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:48:52.554 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:52 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:52.554 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:52 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:52.554 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:52 smithi019 ceph-mon[88137]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:48:52.554 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:52 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:48:52.555 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:52 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:48:52.555 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:52 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:52.555 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:52 smithi019 ceph-mon[88137]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:48:52.555 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:52 smithi019 ceph-mon[88137]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:48:52.555 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:52 smithi019 ceph-mon[88137]: from='client.44166 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:48:52.555 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:48:52.555 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (3m) 19s ago 24m 19.7M - 0.25.0 c8568f914cd2 fec9255f35e7 2024-02-07T12:48:52.555 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (2m) 42s ago 24m 82.9M - 9.4.12 a0ea5e3c46d9 aaffac63bc28 2024-02-07T12:48:52.555 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 unknown 19s ago 24m 63.2M - 3.5 e1d6a67b021e 5113aa77e3f2 2024-02-07T12:48:52.555 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283,8765 running (4m) 42s ago 29m 447M - 19.0.0-1252-g44458cce 79957b372307 5aa0c04fdfd7 2024-02-07T12:48:52.555 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (14m) 19s ago 31m 494M - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:48:52.555 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (58s) 19s ago 31m 41.2M 2048M 19.0.0-1252-g44458cce 79957b372307 8b1b0e21b1c5 2024-02-07T12:48:52.555 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (107s) 42s ago 29m 37.0M 2048M 19.0.0-1252-g44458cce 79957b372307 00f454bbac01 2024-02-07T12:48:52.556 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (30s) 19s ago 29m 23.0M 2048M 19.0.0-1252-g44458cce 79957b372307 6214003f6e08 2024-02-07T12:48:52.556 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (3m) 19s ago 25m 8476k - 1.5.0 0da6a335fe13 c4bdcb1c4d49 2024-02-07T12:48:52.556 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (3m) 42s ago 25m 16.3M - 1.5.0 0da6a335fe13 04ea81f79d85 2024-02-07T12:48:52.556 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (29m) 19s ago 29m 54.7M 2993M 17.2.0 e1d6a67b021e 87f2560a9b15 2024-02-07T12:48:52.556 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (28m) 19s ago 28m 57.2M 2993M 17.2.0 e1d6a67b021e 0f9a190f0e0d 2024-02-07T12:48:52.556 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (28m) 19s ago 28m 53.2M 2993M 17.2.0 e1d6a67b021e 7db94f25a1aa 2024-02-07T12:48:52.556 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (28m) 19s ago 28m 53.5M 2993M 17.2.0 e1d6a67b021e 0b6b674188ed 2024-02-07T12:48:52.556 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (27m) 42s ago 27m 55.7M 3248M 17.2.0 e1d6a67b021e df72cfd69b3b 2024-02-07T12:48:52.556 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (27m) 42s ago 27m 52.4M 3248M 17.2.0 e1d6a67b021e 3bdb22c07537 2024-02-07T12:48:52.556 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (26m) 42s ago 26m 52.1M 3248M 17.2.0 e1d6a67b021e 7859b651e0d9 2024-02-07T12:48:52.556 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (26m) 42s ago 26m 55.5M 3248M 17.2.0 e1d6a67b021e b301e7c75d73 2024-02-07T12:48:52.556 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (3m) 42s ago 25m 61.0M - 2.43.0 a07b618ecd1d c9b607eea1be 2024-02-07T12:48:52.556 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (24m) 19s ago 24m 78.9M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:48:52.556 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (24m) 42s ago 24m 78.3M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:48:52.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:52 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:52.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:52 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:52.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:52 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:48:52.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:52 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:48:52.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:52 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:48:52.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:52 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:52.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:52 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:48:52.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:52 smithi172 ceph-mon[67636]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:48:52.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:52 smithi172 ceph-mon[67636]: from='client.44166 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:48:52.974 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:48:52.974 INFO:teuthology.orchestra.run.smithi019.stdout: "mon": { 2024-02-07T12:48:52.974 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 3 2024-02-07T12:48:52.974 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:48:52.974 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr": { 2024-02-07T12:48:52.974 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:48:52.974 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:48:52.974 INFO:teuthology.orchestra.run.smithi019.stdout: "osd": { 2024-02-07T12:48:52.974 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-02-07T12:48:52.974 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:48:52.974 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw": { 2024-02-07T12:48:52.974 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:48:52.974 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:48:52.975 INFO:teuthology.orchestra.run.smithi019.stdout: "overall": { 2024-02-07T12:48:52.975 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-02-07T12:48:52.975 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 5 2024-02-07T12:48:52.975 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:48:52.975 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:48:53.332 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:48:53.065+0000 7fb71d671700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:48:53.332 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:48:53.333 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:48:53.333 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:48:53.333 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:48:53.333 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:53.333 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:53.333 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:48:53.334 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:48:53.334 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:48:53.334 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:48:53.334 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:48:53.334 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:48:53.334 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:48:53.334 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:48:53.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:48:53.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:48:53.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:48:53.335 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:48:53.336 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:48:53.336 INFO:teuthology.orchestra.run.smithi019.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:48:53.336 INFO:teuthology.orchestra.run.smithi019.stdout: "in_progress": true, 2024-02-07T12:48:53.336 INFO:teuthology.orchestra.run.smithi019.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi019", 2024-02-07T12:48:53.336 INFO:teuthology.orchestra.run.smithi019.stdout: "services_complete": [ 2024-02-07T12:48:53.337 INFO:teuthology.orchestra.run.smithi019.stdout: "mon" 2024-02-07T12:48:53.337 INFO:teuthology.orchestra.run.smithi019.stdout: ], 2024-02-07T12:48:53.337 INFO:teuthology.orchestra.run.smithi019.stdout: "progress": "2/2 daemons upgraded", 2024-02-07T12:48:53.337 INFO:teuthology.orchestra.run.smithi019.stdout: "message": "Currently upgrading mon daemons", 2024-02-07T12:48:53.337 INFO:teuthology.orchestra.run.smithi019.stdout: "is_paused": false 2024-02-07T12:48:53.337 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:48:53.338 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:53.338 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:53.338 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:48:53.338 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:48:53.338 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:48:53.338 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:48:53.339 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:48:53.339 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:48:53.339 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:48:53.339 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:48:53.339 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:48:53.339 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:48:53.340 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:48:53.340 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:48:53.340 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:48:53.340 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:48:53.340 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:48:53.340 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:48:53.341 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:48:53.341 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:48:53.341 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:48:53.341 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:48:53.341 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:48:53.341 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:48:53.342 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:53.342 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:53.342 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:48:53.342 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:48:53.342 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:48:53.342 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:48:53.343 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:48:53.343 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:48:53.343 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:48:53.343 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:48:53.343 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:48:53.343 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:48:53.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:48:53.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:48:53.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:53.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:53.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-mon[88137]: from='client.54109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:48:53.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-mon[88137]: from='client.44178 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:48:53.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/1800787295' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:48:53.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:48:53.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:48:53.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:53.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-mon[82918]: from='client.54109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:48:53.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-mon[82918]: from='client.44178 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:48:53.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/1800787295' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:48:53.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:48:53.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:48:53.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:53.701 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:53 smithi172 ceph-mon[67636]: from='client.54109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:48:53.702 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:53 smithi172 ceph-mon[67636]: from='client.44178 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:48:53.702 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:53 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/1800787295' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:48:53.702 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:53 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:48:53.702 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:53 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:48:53.702 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:53 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:54.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:48:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:48:53] "GET /metrics HTTP/1.1" 200 38018 "" "Prometheus/2.43.0" 2024-02-07T12:48:54.680 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:54 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:48:54.680 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:48:54.680 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:48:54.680 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:48:54.681 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:48:54.681 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:54.681 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:54.681 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:48:54.681 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:48:54.681 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:48:54.681 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:48:54.682 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:48:54.682 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:48:54.682 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:48:54.682 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:48:54.682 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:48:54.682 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:48:54.682 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:48:54.682 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:48:54.683 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:54.683 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:54.683 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:48:54.683 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:48:54.683 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:48:54.683 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:48:54.683 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:48:54.684 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:48:54.684 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:48:54.684 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:48:54.684 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:48:54.684 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:48:54.684 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:48:54.685 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:48:54.685 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:48:54.685 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:48:54.685 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:48:54.685 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:48:54.685 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:48:54.685 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:48:54.686 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:48:54.686 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:48:54.686 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:48:54.686 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:48:54.686 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:54.686 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:54.686 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:48:54.686 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:48:54.687 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:48:54.687 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:48:54.687 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:48:54.687 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:48:54.687 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:48:54.687 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:48:54.687 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:48:54.688 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:48:54.688 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:48:54.688 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:48:54.688 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:54.688 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:54.688 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:54 smithi172 ceph-mon[67636]: Reconfiguring mon.b (monmap changed)... 2024-02-07T12:48:54.689 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:54 smithi172 ceph-mon[67636]: Reconfiguring daemon mon.b on smithi172 2024-02-07T12:48:54.689 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:54 smithi172 ceph-mon[67636]: from='client.44190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:48:54.689 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:54 smithi172 ceph-mon[67636]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:48:54.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:54 smithi019 ceph-mon[88137]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:48:54.844 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:48:54.844 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:48:54.844 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:48:54.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:48:54.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:54.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:54.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:48:54.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:48:54.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:48:54.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:48:54.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:48:54.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:48:54.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:48:54.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:48:54.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:48:54.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:48:54.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:48:54.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:48:54.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:54.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:54.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:48:54.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:48:54.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:48:54.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:48:54.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:48:54.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:48:54.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:48:54.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:48:54.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:48:54.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:48:54.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:48:54.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:48:54.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:48:54.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:48:54.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:48:54.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:48:54.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:48:54.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:48:54.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:48:54.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:48:54.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:48:54.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:48:54.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:54.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:54.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:48:54.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:48:54.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:48:54.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:48:54.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:48:54.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:48:54.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:48:54.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:48:54.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:48:54.848 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:54 smithi019 ceph-mon[82918]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:48:54.848 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:48:54.848 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:48:54.848 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:48:54.848 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:48:54.848 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:54.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:54.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:48:54.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:48:54.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:48:54.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:48:54.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:48:54.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:48:54.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:48:54.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:48:54.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:48:54.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:48:54.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:48:54.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:48:54.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:54.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:54.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:48:54.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:48:54.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:48:54.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:48:54.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:48:54.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:48:54.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:48:54.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:48:54.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:48:54.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:48:54.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:48:54.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:48:54.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:48:54.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:48:54.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:48:54.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:48:54.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:48:54.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:48:54.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:48:54.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:48:54.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:48:54.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:48:54.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:54.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:54.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:48:54.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:48:54.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:48:54.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:48:54.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:48:54.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:48:54.852 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:48:54.852 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:48:54.852 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:48:54.852 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:48:54.852 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:48:54.852 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:48:54.852 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:54.852 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:54.852 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:54 smithi019 ceph-mon[88137]: Reconfiguring mon.b (monmap changed)... 2024-02-07T12:48:54.852 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:54 smithi019 ceph-mon[88137]: Reconfiguring daemon mon.b on smithi172 2024-02-07T12:48:54.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:54 smithi019 ceph-mon[88137]: from='client.44190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:48:54.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:54 smithi019 ceph-mon[88137]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:48:54.853 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:48:54.853 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:48:54.853 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:48:54.853 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:48:54.853 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:48:54.853 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:54 smithi019 ceph-mon[82918]: Reconfiguring mon.b (monmap changed)... 2024-02-07T12:48:54.853 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:54 smithi019 ceph-mon[82918]: Reconfiguring daemon mon.b on smithi172 2024-02-07T12:48:54.853 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:54 smithi019 ceph-mon[82918]: from='client.44190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:48:54.853 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:54 smithi019 ceph-mon[82918]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:48:56.037 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:55 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:56.037 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:55 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:56.038 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:55 smithi172 ceph-mon[67636]: Reconfiguring mgr.x (monmap changed)... 2024-02-07T12:48:56.038 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:55 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:48:56.038 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:55 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:48:56.038 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:55 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:48:56.038 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:55 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:56.038 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:55 smithi172 ceph-mon[67636]: Reconfiguring daemon mgr.x on smithi172 2024-02-07T12:48:56.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:55 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:56.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:55 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:56.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:55 smithi019 ceph-mon[88137]: Reconfiguring mgr.x (monmap changed)... 2024-02-07T12:48:56.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:55 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:48:56.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:55 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:48:56.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:55 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:48:56.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:55 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:56.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:55 smithi019 ceph-mon[88137]: Reconfiguring daemon mgr.x on smithi172 2024-02-07T12:48:56.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:55 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:56.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:55 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:56.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:55 smithi019 ceph-mon[82918]: Reconfiguring mgr.x (monmap changed)... 2024-02-07T12:48:56.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:55 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:48:56.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:55 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:48:56.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:55 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:48:56.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:55 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:56.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:55 smithi019 ceph-mon[82918]: Reconfiguring daemon mgr.x on smithi172 2024-02-07T12:48:57.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:56 smithi019 ceph-mon[82918]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:48:57.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:56 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:57.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:56 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:57.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:56 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-02-07T12:48:57.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:56 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:57.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:56 smithi019 ceph-mon[88137]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:48:57.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:56 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:57.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:56 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:57.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:56 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-02-07T12:48:57.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:56 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:57.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:56 smithi172 ceph-mon[67636]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:48:57.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:56 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:57.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:56 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:57.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:56 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-02-07T12:48:57.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:56 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:58.092 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:57 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:48:58.092 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:57 smithi172 ceph-mon[67636]: Reconfiguring osd.4 (monmap changed)... 2024-02-07T12:48:58.092 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:57 smithi172 ceph-mon[67636]: Reconfiguring daemon osd.4 on smithi172 2024-02-07T12:48:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:57 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:48:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:57 smithi019 ceph-mon[88137]: Reconfiguring osd.4 (monmap changed)... 2024-02-07T12:48:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:57 smithi019 ceph-mon[88137]: Reconfiguring daemon osd.4 on smithi172 2024-02-07T12:48:58.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:57 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:48:58.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:57 smithi019 ceph-mon[82918]: Reconfiguring osd.4 (monmap changed)... 2024-02-07T12:48:58.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:57 smithi019 ceph-mon[82918]: Reconfiguring daemon osd.4 on smithi172 2024-02-07T12:48:59.089 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:58 smithi172 ceph-mon[67636]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:48:59.089 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:58 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:59.089 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:58 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:59.089 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:58 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-02-07T12:48:59.089 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:58 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:59.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:58 smithi019 ceph-mon[82918]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:48:59.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:58 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:59.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:58 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:59.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:58 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-02-07T12:48:59.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:58 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:48:59.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:58 smithi019 ceph-mon[88137]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:48:59.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:58 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:59.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:58 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:48:59.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:58 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-02-07T12:48:59.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:58 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:00.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:59 smithi172 ceph-mon[67636]: Reconfiguring osd.5 (monmap changed)... 2024-02-07T12:49:00.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:59 smithi172 ceph-mon[67636]: Reconfiguring daemon osd.5 on smithi172 2024-02-07T12:49:00.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:59 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:00.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:48:59 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:00.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:59 smithi019 ceph-mon[88137]: Reconfiguring osd.5 (monmap changed)... 2024-02-07T12:49:00.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:59 smithi019 ceph-mon[88137]: Reconfiguring daemon osd.5 on smithi172 2024-02-07T12:49:00.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:59 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:00.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:48:59 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:00.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:59 smithi019 ceph-mon[82918]: Reconfiguring osd.5 (monmap changed)... 2024-02-07T12:49:00.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:59 smithi019 ceph-mon[82918]: Reconfiguring daemon osd.5 on smithi172 2024-02-07T12:49:00.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:59 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:00.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:48:59 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:01.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:00 smithi172 ceph-mon[67636]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:01.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:00 smithi172 ceph-mon[67636]: Reconfiguring osd.6 (monmap changed)... 2024-02-07T12:49:01.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:00 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-02-07T12:49:01.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:00 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:01.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:00 smithi172 ceph-mon[67636]: Reconfiguring daemon osd.6 on smithi172 2024-02-07T12:49:01.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:00 smithi019 ceph-mon[88137]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:01.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:00 smithi019 ceph-mon[88137]: Reconfiguring osd.6 (monmap changed)... 2024-02-07T12:49:01.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:00 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-02-07T12:49:01.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:00 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:01.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:00 smithi019 ceph-mon[88137]: Reconfiguring daemon osd.6 on smithi172 2024-02-07T12:49:01.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:00 smithi019 ceph-mon[82918]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:01.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:00 smithi019 ceph-mon[82918]: Reconfiguring osd.6 (monmap changed)... 2024-02-07T12:49:01.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:00 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-02-07T12:49:01.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:00 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:01.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:00 smithi019 ceph-mon[82918]: Reconfiguring daemon osd.6 on smithi172 2024-02-07T12:49:02.832 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:02 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:02.832 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:02 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:02.832 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:02 smithi172 ceph-mon[67636]: Reconfiguring osd.7 (monmap changed)... 2024-02-07T12:49:02.832 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:02 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-02-07T12:49:02.832 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:02 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:02.832 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:02 smithi172 ceph-mon[67636]: Reconfiguring daemon osd.7 on smithi172 2024-02-07T12:49:02.833 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:02 smithi172 ceph-mon[67636]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:03.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:02 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:03.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:02 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:03.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:02 smithi019 ceph-mon[88137]: Reconfiguring osd.7 (monmap changed)... 2024-02-07T12:49:03.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:02 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-02-07T12:49:03.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:02 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:03.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:02 smithi019 ceph-mon[88137]: Reconfiguring daemon osd.7 on smithi172 2024-02-07T12:49:03.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:02 smithi019 ceph-mon[88137]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:03.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:02 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:03.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:02 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:03.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:02 smithi019 ceph-mon[82918]: Reconfiguring osd.7 (monmap changed)... 2024-02-07T12:49:03.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:02 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-02-07T12:49:03.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:02 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:03.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:02 smithi019 ceph-mon[82918]: Reconfiguring daemon osd.7 on smithi172 2024-02-07T12:49:03.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:02 smithi019 ceph-mon[82918]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:04.243 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:49:03] "GET /metrics HTTP/1.1" 200 38018 "" "Prometheus/2.43.0" 2024-02-07T12:49:04.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:04 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:04.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:04 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:04.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:04 smithi019 ceph-mon[82918]: Reconfiguring rgw.foo.smithi172.wueguy (monmap changed)... 2024-02-07T12:49:04.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:04 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.wueguy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:49:04.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:04 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.wueguy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:49:04.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:04 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:04.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:04 smithi019 ceph-mon[82918]: Reconfiguring daemon rgw.foo.smithi172.wueguy on smithi172 2024-02-07T12:49:04.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:04 smithi019 ceph-mon[82918]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:04.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:04 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:04.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:04 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:04.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:04 smithi019 ceph-mon[88137]: Reconfiguring rgw.foo.smithi172.wueguy (monmap changed)... 2024-02-07T12:49:04.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:04 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.wueguy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:49:04.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:04 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.wueguy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:49:04.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:04 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:04.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:04 smithi019 ceph-mon[88137]: Reconfiguring daemon rgw.foo.smithi172.wueguy on smithi172 2024-02-07T12:49:04.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:04 smithi019 ceph-mon[88137]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:04.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:04 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:04.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:04 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:04.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:04 smithi172 ceph-mon[67636]: Reconfiguring rgw.foo.smithi172.wueguy (monmap changed)... 2024-02-07T12:49:04.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:04 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.wueguy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:49:04.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:04 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.wueguy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:49:04.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:04 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:04.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:04 smithi172 ceph-mon[67636]: Reconfiguring daemon rgw.foo.smithi172.wueguy on smithi172 2024-02-07T12:49:04.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:04 smithi172 ceph-mon[67636]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:06.102 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.102 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.102 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:49:06.102 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.102 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.102 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.102 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.102 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.102 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-02-07T12:49:06.103 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.103 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.103 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-02-07T12:49:06.103 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.103 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.103 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.103 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.103 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.103 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.103 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.103 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.103 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.103 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.104 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.104 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.104 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:49:06.105 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.105 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.105 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.105 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.105 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.105 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-02-07T12:49:06.105 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.105 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.105 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-02-07T12:49:06.105 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.105 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.105 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.106 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.106 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.106 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.106 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.106 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.106 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.106 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.106 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.106 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.106 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.106 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.106 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.106 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.106 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.106 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.107 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:49:06.107 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.107 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.107 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.107 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.107 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.107 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.107 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.107 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.107 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.108 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.108 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.108 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.108 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.108 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.108 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.108 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.108 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.108 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.108 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.108 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.108 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.108 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.108 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.109 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:49:06.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.109 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.110 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.111 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.111 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.111 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.111 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.111 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.111 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:49:06.111 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:49:06.111 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-02-07T12:49:06.111 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.111 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.111 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-02-07T12:49:06.111 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:49:06.112 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:49:06.112 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-02-07T12:49:06.112 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:49:06.112 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:49:06.112 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:49:06.112 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:49:06.112 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-02-07T12:49:06.112 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:49:06.112 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:49:06.112 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:49:06.112 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:49:06.112 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-02-07T12:49:06.112 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:49:06.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:49:06.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-02-07T12:49:06.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-02-07T12:49:06.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:49:06.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:49:06.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-02-07T12:49:06.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:49:06.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:49:06.113 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:49:06.114 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:49:06.114 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-02-07T12:49:06.114 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:49:06.114 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:49:06.114 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:49:06.114 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:49:06.114 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-02-07T12:49:06.114 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.114 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.115 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:49:06.115 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:49:06.115 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-02-07T12:49:06.115 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:49:06.115 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:49:06.115 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:49:06.115 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:49:06.115 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-02-07T12:49:06.115 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:49:06.115 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:49:06.115 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-02-07T12:49:06.115 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.115 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.115 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.115 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.115 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.115 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.116 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.116 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.116 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.116 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.116 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.116 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.116 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.116 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:49:06.116 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:49:06.116 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-02-07T12:49:06.116 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:49:06.116 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:49:06.117 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:49:06.117 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:49:06.117 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-02-07T12:49:06.117 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:49:06.117 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:49:06.117 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-02-07T12:49:06.117 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.117 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.117 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.117 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.117 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.117 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.117 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.117 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.117 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.118 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.118 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.118 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.118 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:49:06.118 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:49:06.118 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-02-07T12:49:06.118 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:49:06.118 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:06.118 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:49:06.118 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.118 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:49:06.119 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:49:06.119 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:06.119 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.119 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:49:06.119 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:49:06.119 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-02-07T12:49:06.119 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:49:06.119 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:06.119 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:49:06.119 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.119 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:49:06.119 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:49:06.119 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:06.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:49:06.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-02-07T12:49:06.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-02-07T12:49:06.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:49:06.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:06.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-02-07T12:49:06.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-02-07T12:49:06.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:49:06.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:49:06.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-02-07T12:49:06.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-02-07T12:49:06.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:49:06.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:49:06.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-02-07T12:49:06.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:49:06.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:49:06.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:49:06.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:49:06.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-02-07T12:49:06.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:49:06.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:49:06.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:49:06.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:49:06.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-02-07T12:49:06.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:49:06.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:49:06.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-02-07T12:49:06.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:49:06.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:49:06.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:49:06.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:49:06.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-02-07T12:49:06.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:49:06.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:49:06.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-02-07T12:49:06.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.377 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.377 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:49:06.377 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:49:06.377 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:49:06.377 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-02-07T12:49:06.377 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:49:06.377 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:06.377 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:49:06.377 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:06.377 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:49:06.377 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:49:06.377 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:07.219 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all mon 2024-02-07T12:49:07.219 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all crash 2024-02-07T12:49:07.219 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all mds 2024-02-07T12:49:07.219 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all rbd-mirror 2024-02-07T12:49:07.219 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all cephfs-mirror 2024-02-07T12:49:07.219 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all ceph-exporter 2024-02-07T12:49:07.220 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all nfs 2024-02-07T12:49:07.220 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all nvmeof 2024-02-07T12:49:07.220 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all node-exporter 2024-02-07T12:49:07.220 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all prometheus 2024-02-07T12:49:07.220 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all alertmanager 2024-02-07T12:49:07.220 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all grafana 2024-02-07T12:49:07.220 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all loki 2024-02-07T12:49:07.220 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all promtail 2024-02-07T12:49:07.220 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[88137]: Upgrade: Finalizing container_image settings 2024-02-07T12:49:07.220 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[88137]: Upgrade: Complete! 2024-02-07T12:49:07.220 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[88137]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 892 B/s rd, 0 op/s 2024-02-07T12:49:07.220 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[88137]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:49:07.221 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[88137]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:49:07.221 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[88137]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:49:07.221 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:49:07.221 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all mon 2024-02-07T12:49:07.221 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all crash 2024-02-07T12:49:07.221 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all mds 2024-02-07T12:49:07.221 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all rbd-mirror 2024-02-07T12:49:07.221 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all cephfs-mirror 2024-02-07T12:49:07.221 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all ceph-exporter 2024-02-07T12:49:07.221 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all nfs 2024-02-07T12:49:07.221 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all nvmeof 2024-02-07T12:49:07.222 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all node-exporter 2024-02-07T12:49:07.222 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all prometheus 2024-02-07T12:49:07.222 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all alertmanager 2024-02-07T12:49:07.222 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all grafana 2024-02-07T12:49:07.222 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all loki 2024-02-07T12:49:07.222 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all promtail 2024-02-07T12:49:07.222 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[82918]: Upgrade: Finalizing container_image settings 2024-02-07T12:49:07.222 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[82918]: Upgrade: Complete! 2024-02-07T12:49:07.222 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[82918]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 892 B/s rd, 0 op/s 2024-02-07T12:49:07.222 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[82918]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:49:07.222 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[82918]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:49:07.222 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[82918]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:49:07.222 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:06 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:49:07.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:06 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all mon 2024-02-07T12:49:07.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:06 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all crash 2024-02-07T12:49:07.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:06 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all mds 2024-02-07T12:49:07.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:06 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all rbd-mirror 2024-02-07T12:49:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:06 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all cephfs-mirror 2024-02-07T12:49:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:06 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all ceph-exporter 2024-02-07T12:49:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:06 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all nfs 2024-02-07T12:49:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:06 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all nvmeof 2024-02-07T12:49:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:06 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all node-exporter 2024-02-07T12:49:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:06 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all prometheus 2024-02-07T12:49:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:06 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all alertmanager 2024-02-07T12:49:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:06 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all grafana 2024-02-07T12:49:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:06 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all loki 2024-02-07T12:49:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:06 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all promtail 2024-02-07T12:49:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:06 smithi172 ceph-mon[67636]: Upgrade: Finalizing container_image settings 2024-02-07T12:49:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:06 smithi172 ceph-mon[67636]: Upgrade: Complete! 2024-02-07T12:49:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:06 smithi172 ceph-mon[67636]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 892 B/s rd, 0 op/s 2024-02-07T12:49:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:06 smithi172 ceph-mon[67636]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:49:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:06 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:49:07.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:06 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:49:07.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:06 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:49:07.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:49:07.215+0000 7fb71d671700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:49:07.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:49:07.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:49:07.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:49:07.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:49:07.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:07.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:07.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:49:07.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:49:07.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:49:07.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:49:07.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:49:07.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:49:07.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:49:07.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:49:07.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:49:07.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:49:07.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:49:07.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:49:07.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:07.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:07.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:49:07.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:49:07.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:49:07.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:49:07.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:49:07.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:49:07.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:49:07.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:49:07.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:49:07.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:49:07.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:49:07.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:49:07.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:49:07.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:49:07.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:49:07.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:49:07.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:49:07.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:49:07.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:49:07.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:49:07.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:49:07.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:49:07.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:07.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:07.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:49:07.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:49:07.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:49:07.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:49:07.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:49:07.600 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:49:07.600 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:49:07.600 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:49:07.600 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:49:07.600 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:49:07.600 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:49:07.600 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:49:07.600 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:07.600 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:08.300 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-mon[82918]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:49:08.300 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-mon[82918]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:49:08.300 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:49:08.300 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:49:08.300 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:49:08.300 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:49:08.300 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:08.300 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:08.300 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:49:08.301 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:49:08.301 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:49:08.301 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:49:08.301 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:49:08.301 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:49:08.301 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:49:08.301 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:49:08.301 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:49:08.301 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:49:08.301 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:49:08.301 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:49:08.301 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:08.301 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:08.301 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:49:08.301 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:49:08.302 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:49:08.302 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:49:08.302 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:49:08.302 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:49:08.302 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:49:08.302 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:49:08.302 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:49:08.302 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:49:08.302 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:49:08.302 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:49:08.302 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:49:08.302 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:49:08.302 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:49:08.302 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:49:08.302 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:49:08.303 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:49:08.303 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:49:08.303 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:49:08.303 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:49:08.303 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:49:08.303 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:08.303 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:08.303 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:49:08.303 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:49:08.303 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:49:08.303 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:49:08.303 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:49:08.303 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:49:08.303 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:49:08.303 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:49:08.304 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:49:08.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-mon[88137]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:49:08.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-mon[88137]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:49:08.304 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:49:08.304 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:49:08.304 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:49:08.304 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:49:08.304 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:08.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:08.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:49:08.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:49:08.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:49:08.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:49:08.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:49:08.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:49:08.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:49:08.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:49:08.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:49:08.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:49:08.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:49:08.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:49:08.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:08.305 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:08.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:49:08.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:49:08.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:49:08.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:49:08.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:49:08.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:49:08.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:49:08.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:49:08.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:49:08.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:49:08.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:49:08.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:49:08.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:49:08.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:49:08.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:49:08.306 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:49:08.307 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:49:08.307 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:49:08.307 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:49:08.307 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:49:08.307 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:49:08.307 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:49:08.307 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:08.307 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:08.307 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:49:08.307 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:49:08.307 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:49:08.307 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:49:08.307 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:49:08.307 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:49:08.307 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:49:08.308 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:49:08.308 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:49:08.308 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:49:08.308 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:49:08.308 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:49:08.308 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:08.308 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:08.308 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:49:08.308 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:08.308 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:49:08.308 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-mon[88137]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-02-07T12:49:08.308 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-mon[88137]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 927 B/s rd, 0 op/s 2024-02-07T12:49:08.309 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:08.309 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-mon[88137]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:49:08.309 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:49:08.309 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:49:08.309 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:08.309 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-mon[88137]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:49:08.309 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:49:08.309 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:49:08.309 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:49:08.309 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:08.309 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:08.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:49:08.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:08.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:49:08.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-mon[82918]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-02-07T12:49:08.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-mon[82918]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 927 B/s rd, 0 op/s 2024-02-07T12:49:08.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:08.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-mon[82918]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:49:08.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:49:08.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:49:08.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:08.310 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:07 smithi019 ceph-mon[82918]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:49:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:07 smithi172 ceph-mon[67636]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:49:08.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:07 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:49:08.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:49:08.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:49:08.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:49:08.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:49:08.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:08.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:08.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:49:08.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:49:08.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:49:08.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:49:08.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:49:08.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:49:08.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:49:08.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:49:08.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:49:08.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:49:08.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:49:08.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:49:08.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:08.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:08.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:49:08.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:49:08.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:49:08.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:49:08.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:49:08.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:49:08.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:49:08.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:49:08.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:49:08.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:49:08.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:49:08.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:49:08.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:49:08.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:49:08.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:49:08.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:49:08.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:49:08.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:49:08.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:49:08.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:49:08.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:49:08.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:49:08.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:08.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:08.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:49:08.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:49:08.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:49:08.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:49:08.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:49:08.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:49:08.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:49:08.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:49:08.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:49:08.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:49:08.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:49:08.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:49:08.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:08.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:08.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:07 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:49:08.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:07 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:08.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:07 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:49:08.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:07 smithi172 ceph-mon[67636]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-02-07T12:49:08.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:07 smithi172 ceph-mon[67636]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 927 B/s rd, 0 op/s 2024-02-07T12:49:08.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:07 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:08.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:07 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:49:08.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:07 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:49:08.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:07 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:49:08.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:07 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:08.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:07 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:49:09.359 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:49:09.232+0000 7fb71d671700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:49:09.359 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:49:09.359 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:49:09.359 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:49:09.359 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:49:09.359 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:09.359 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:09.359 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:49:09.360 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:49:09.360 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:49:09.360 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:49:09.360 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:49:09.360 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:49:09.360 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:49:09.360 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:49:09.360 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:49:09.360 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:49:09.360 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:49:09.360 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:49:09.364 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:09.364 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:09.364 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:49:09.364 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:49:09.364 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:49:09.364 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:49:09.364 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:49:09.364 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:49:09.364 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:49:09.364 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:49:09.364 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:49:09.365 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:49:09.365 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:49:09.365 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:49:09.365 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:49:09.365 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:49:09.365 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:49:09.365 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:49:09.365 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:49:09.365 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:49:09.365 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:49:09.365 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:49:09.365 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:49:09.365 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:49:09.366 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:09.366 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:09.366 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:49:09.366 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:49:09.366 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:49:09.366 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:49:09.366 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:49:09.366 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:49:09.366 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:49:09.366 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:49:09.366 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:49:09.366 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:49:09.366 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:49:09.366 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:49:09.366 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:09.367 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:09 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:11.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:10 smithi172 ceph-mon[67636]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 927 B/s rd, 0 op/s 2024-02-07T12:49:11.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:10 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:49:11.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:49:11.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:49:11.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:49:11.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:49:11.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:11.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:11.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:49:11.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:49:11.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:49:11.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:49:11.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:49:11.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:49:11.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:49:11.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:49:11.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:49:11.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:49:11.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:49:11.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:49:11.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:11.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:11.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:49:11.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:49:11.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:49:11.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:49:11.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:49:11.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:49:11.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:49:11.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:49:11.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:49:11.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:49:11.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:49:11.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:49:11.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:49:11.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:49:11.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:49:11.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:49:11.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:49:11.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:49:11.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:49:11.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:49:11.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:49:11.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:49:11.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:11.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:11.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:49:11.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:49:11.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:49:11.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:49:11.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:49:11.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:49:11.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:49:11.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:49:11.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:49:11.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:49:11.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:49:11.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:49:11.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:11.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:11.123 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:10 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:11.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:10 smithi019 ceph-mon[82918]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 927 B/s rd, 0 op/s 2024-02-07T12:49:11.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:10 smithi019 ceph-mon[82918]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:49:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:49:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:49:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:49:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:49:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:49:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:49:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:49:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:49:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:49:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:49:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:49:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:49:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:49:11.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:49:11.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:49:11.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:49:11.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:11.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:11.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:49:11.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:49:11.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:49:11.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:49:11.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:49:11.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:49:11.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:49:11.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:49:11.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:49:11.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:49:11.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:49:11.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:49:11.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:49:11.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:49:11.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:49:11.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:49:11.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:49:11.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:49:11.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:49:11.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:49:11.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:49:11.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:49:11.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:11.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:11.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:49:11.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:49:11.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:49:11.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:49:11.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:49:11.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:49:11.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:49:11.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:49:11.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:49:11.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:49:11.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:49:11.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:49:11.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:11.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:11.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:10 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:11.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:10 smithi019 ceph-mon[88137]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 927 B/s rd, 0 op/s 2024-02-07T12:49:11.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:10 smithi019 ceph-mon[88137]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:49:11.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:49:11.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:49:11.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:49:11.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:49:11.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:11.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:11.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:49:11.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:49:11.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:49:11.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:49:11.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:49:11.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:49:11.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:49:11.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:49:11.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:49:11.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:49:11.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:49:11.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:49:11.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:11.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:11.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:49:11.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:49:11.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:49:11.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:49:11.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:49:11.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:49:11.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:49:11.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:49:11.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:49:11.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:49:11.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:49:11.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:49:11.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:49:11.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:49:11.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:49:11.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:49:11.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:49:11.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:49:11.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:49:11.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:49:11.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:49:11.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:49:11.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:11.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:11.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:49:11.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:49:11.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:49:11.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:49:11.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:49:11.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:49:11.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:49:11.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:49:11.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:49:11.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:49:11.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:49:11.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:49:11.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:11.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:11.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:10 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:13.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:12 smithi172 ceph-mon[67636]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-02-07T12:49:13.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:12 smithi019 ceph-mon[88137]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-02-07T12:49:13.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:12 smithi019 ceph-mon[82918]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-02-07T12:49:14.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:49:13] "GET /metrics HTTP/1.1" 200 37948 "" "Prometheus/2.43.0" 2024-02-07T12:49:15.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:14 smithi172 ceph-mon[67636]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:15.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:14 smithi019 ceph-mon[88137]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:15.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:14 smithi019 ceph-mon[82918]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:17.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:16 smithi172 ceph-mon[67636]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 635 B/s rd, 0 op/s 2024-02-07T12:49:17.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:16 smithi019 ceph-mon[82918]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 635 B/s rd, 0 op/s 2024-02-07T12:49:17.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:16 smithi019 ceph-mon[88137]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 635 B/s rd, 0 op/s 2024-02-07T12:49:18.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:17 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:49:18.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:17 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:49:18.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:17 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:49:19.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:18 smithi172 ceph-mon[67636]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:49:19.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:18 smithi019 ceph-mon[82918]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:49:19.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:18 smithi019 ceph-mon[88137]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:49:21.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:20 smithi019 ceph-mon[82918]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:21.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:20 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:49:21.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:20 smithi019 ceph-mon[88137]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:21.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:20 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:49:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:20 smithi172 ceph-mon[67636]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:20 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:49:23.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:22 smithi019 ceph-mon[88137]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:23.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:22 smithi019 ceph-mon[82918]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:23.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:22 smithi172 ceph-mon[67636]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:24.005 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:49:23] "GET /metrics HTTP/1.1" 200 38020 "" "Prometheus/2.43.0" 2024-02-07T12:49:24.384 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch ps' 2024-02-07T12:49:24.969 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:24 smithi019 ceph-mon[88137]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:24.969 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:24 smithi019 ceph-mon[88137]: from='client.54133 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:49:24.969 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:24 smithi019 ceph-mon[82918]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:24.969 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:24 smithi019 ceph-mon[82918]: from='client.54133 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:49:25.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:24 smithi172 ceph-mon[67636]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:25.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:24 smithi172 ceph-mon[67636]: from='client.54133 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:49:26.430 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:49:26.431 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (3m) 53s ago 25m 19.7M - 0.25.0 c8568f914cd2 fec9255f35e7 2024-02-07T12:49:26.431 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (3m) 76s ago 25m 82.9M - 9.4.12 a0ea5e3c46d9 aaffac63bc28 2024-02-07T12:49:26.431 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 unknown 53s ago 24m 63.2M - 3.5 e1d6a67b021e 5113aa77e3f2 2024-02-07T12:49:26.431 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283,8765 running (4m) 76s ago 30m 447M - 19.0.0-1252-g44458cce 79957b372307 5aa0c04fdfd7 2024-02-07T12:49:26.431 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (14m) 53s ago 31m 494M - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:49:26.431 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (91s) 53s ago 31m 41.2M 2048M 19.0.0-1252-g44458cce 79957b372307 8b1b0e21b1c5 2024-02-07T12:49:26.431 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (2m) 76s ago 30m 37.0M 2048M 19.0.0-1252-g44458cce 79957b372307 00f454bbac01 2024-02-07T12:49:26.431 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (64s) 53s ago 30m 23.0M 2048M 19.0.0-1252-g44458cce 79957b372307 6214003f6e08 2024-02-07T12:49:26.431 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (4m) 53s ago 26m 8476k - 1.5.0 0da6a335fe13 c4bdcb1c4d49 2024-02-07T12:49:26.431 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (4m) 76s ago 26m 16.3M - 1.5.0 0da6a335fe13 04ea81f79d85 2024-02-07T12:49:26.431 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (29m) 53s ago 29m 54.7M 2993M 17.2.0 e1d6a67b021e 87f2560a9b15 2024-02-07T12:49:26.431 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (29m) 53s ago 29m 57.2M 2993M 17.2.0 e1d6a67b021e 0f9a190f0e0d 2024-02-07T12:49:26.432 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (28m) 53s ago 28m 53.2M 2993M 17.2.0 e1d6a67b021e 7db94f25a1aa 2024-02-07T12:49:26.432 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (28m) 53s ago 28m 53.5M 2993M 17.2.0 e1d6a67b021e 0b6b674188ed 2024-02-07T12:49:26.432 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (28m) 76s ago 28m 55.7M 3248M 17.2.0 e1d6a67b021e df72cfd69b3b 2024-02-07T12:49:26.432 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (27m) 76s ago 27m 52.4M 3248M 17.2.0 e1d6a67b021e 3bdb22c07537 2024-02-07T12:49:26.432 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (27m) 76s ago 27m 52.1M 3248M 17.2.0 e1d6a67b021e 7859b651e0d9 2024-02-07T12:49:26.432 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (27m) 76s ago 27m 55.5M 3248M 17.2.0 e1d6a67b021e b301e7c75d73 2024-02-07T12:49:26.432 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (4m) 76s ago 26m 61.0M - 2.43.0 a07b618ecd1d c9b607eea1be 2024-02-07T12:49:26.432 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (24m) 53s ago 24m 78.9M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:49:26.432 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (24m) 76s ago 24m 78.3M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:49:27.054 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:26 smithi019 ceph-mon[82918]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:27.054 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:26 smithi019 ceph-mon[88137]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:27.078 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 1'"'"'' 2024-02-07T12:49:27.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:26 smithi172 ceph-mon[67636]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:28.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:27 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:49:28.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:27 smithi019 ceph-mon[88137]: from='client.44202 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:49:28.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:27 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:49:28.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:27 smithi019 ceph-mon[82918]: from='client.44202 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:49:28.143 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:27 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:49:28.143 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:27 smithi172 ceph-mon[67636]: from='client.44202 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:49:29.105 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:49:29.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:28 smithi019 ceph-mon[88137]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:29.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:28 smithi019 ceph-mon[82918]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:28 smithi172 ceph-mon[67636]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:29.735 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph versions | jq -e '"'"'.mon | keys'"'"' | grep $sha1' 2024-02-07T12:49:30.051 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:29 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/301554996' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:30.051 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:29 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/301554996' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:30.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:29 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/301554996' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:31.008 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:30 smithi019 ceph-mon[88137]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:31.008 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:30 smithi019 ceph-mon[82918]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:31.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:30 smithi172 ceph-mon[67636]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:31.780 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)" 2024-02-07T12:49:32.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:31 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/3735465598' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:32.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:31 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/3735465598' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:32.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:31 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/3735465598' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:32.473 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 5'"'"'' 2024-02-07T12:49:33.086 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:32 smithi019 ceph-mon[88137]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:33.086 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:32 smithi019 ceph-mon[82918]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:32 smithi172 ceph-mon[67636]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:34.018 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:49:33] "GET /metrics HTTP/1.1" 200 38020 "" "Prometheus/2.43.0" 2024-02-07T12:49:35.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:34 smithi019 ceph-mon[88137]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:35.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:34 smithi019 ceph-mon[82918]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:34 smithi172 ceph-mon[67636]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:35.935 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:35 smithi019 ceph-mon[82918]: from='client.44220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:49:35.935 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:35 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:49:35.936 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:35 smithi019 ceph-mon[88137]: from='client.44220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:49:35.936 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:35 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:49:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:35 smithi172 ceph-mon[67636]: from='client.44220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:49:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:35 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:49:36.589 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:49:37.182 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:36 smithi019 ceph-mon[88137]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:37.182 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:36 smithi019 ceph-mon[82918]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:37.209 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch upgrade status' 2024-02-07T12:49:37.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:36 smithi172 ceph-mon[67636]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:38.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:37 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:49:38.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:37 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:49:38.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:37 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:49:38.950 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:38 smithi019 ceph-mon[88137]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:38.951 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:38 smithi019 ceph-mon[82918]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:39.202 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:49:39.202 INFO:teuthology.orchestra.run.smithi019.stdout: "target_image": null, 2024-02-07T12:49:39.202 INFO:teuthology.orchestra.run.smithi019.stdout: "in_progress": false, 2024-02-07T12:49:39.203 INFO:teuthology.orchestra.run.smithi019.stdout: "which": "", 2024-02-07T12:49:39.203 INFO:teuthology.orchestra.run.smithi019.stdout: "services_complete": [], 2024-02-07T12:49:39.203 INFO:teuthology.orchestra.run.smithi019.stdout: "progress": null, 2024-02-07T12:49:39.203 INFO:teuthology.orchestra.run.smithi019.stdout: "message": "", 2024-02-07T12:49:39.203 INFO:teuthology.orchestra.run.smithi019.stdout: "is_paused": false 2024-02-07T12:49:39.203 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:49:39.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:38 smithi172 ceph-mon[67636]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:39.921 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph health detail' 2024-02-07T12:49:41.204 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:40 smithi019 ceph-mon[88137]: from='client.44226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:49:41.204 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:40 smithi019 ceph-mon[88137]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:41.204 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:40 smithi019 ceph-mon[82918]: from='client.44226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:49:41.205 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:40 smithi019 ceph-mon[82918]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:41.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:40 smithi172 ceph-mon[67636]: from='client.44226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:49:41.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:40 smithi172 ceph-mon[67636]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:41.984 INFO:teuthology.orchestra.run.smithi019.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-02-07T12:49:41.985 INFO:teuthology.orchestra.run.smithi019.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-02-07T12:49:41.985 INFO:teuthology.orchestra.run.smithi019.stdout: daemon iscsi.foo.smithi019.svjimj on smithi019 is in unknown state 2024-02-07T12:49:42.610 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2' 2024-02-07T12:49:43.231 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:42 smithi019 ceph-mon[88137]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:43.231 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:42 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/1812230215' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:49:43.231 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:42 smithi019 ceph-mon[82918]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:43.231 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:42 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/1812230215' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:49:43.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:42 smithi172 ceph-mon[67636]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:43.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:42 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/1812230215' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:49:44.175 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:49:43] "GET /metrics HTTP/1.1" 200 38017 "" "Prometheus/2.43.0" 2024-02-07T12:49:45.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:44 smithi019 ceph-mon[82918]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:45.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:44 smithi019 ceph-mon[88137]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:45.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:44 smithi172 ceph-mon[67636]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:46.071 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:45 smithi019 ceph-mon[82918]: from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:49:46.072 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:45 smithi019 ceph-mon[88137]: from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:49:46.272 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:45 smithi172 ceph-mon[67636]: from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:49:46.692 INFO:teuthology.orchestra.run.smithi019.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:49:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:46 smithi172 ceph-mon[67636]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:46 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:47.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:46 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:49:47.279 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:46 smithi019 ceph-mon[88137]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:47.279 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:46 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:47.279 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:46 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:49:47.279 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:46 smithi019 ceph-mon[82918]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:49:47.279 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:46 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:47.279 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:46 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:49:47.304 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-02-07T12:49:48.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:47 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:49:48.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:47 smithi019 ceph-mon[88137]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:49:48.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:47 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:48.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:47 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:48.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:47 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:48.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:47 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:48.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:47 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:48.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:47 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:49:48.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:47 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:47 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:49:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:47 smithi019 ceph-mon[82918]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:49:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:47 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:47 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:47 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:47 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:47 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:47 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:49:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:47 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:48.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:47 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:49:48.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:47 smithi172 ceph-mon[67636]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:49:48.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:47 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:48.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:47 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:48.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:47 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:48.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:47 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:48.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:47 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:48.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:47 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:49:48.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:47 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:49.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:48 smithi019 ceph-mon[88137]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:49.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:48 smithi019 ceph-mon[88137]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 961 B/s rd, 0 op/s 2024-02-07T12:49:49.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:48 smithi019 ceph-mon[88137]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:49.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:49:49.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:49:49.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:49.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:48 smithi019 ceph-mon[82918]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:49.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:48 smithi019 ceph-mon[82918]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 961 B/s rd, 0 op/s 2024-02-07T12:49:49.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:48 smithi019 ceph-mon[82918]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:49.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:49:49.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:49:49.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:49.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:48 smithi172 ceph-mon[67636]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:49.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:48 smithi172 ceph-mon[67636]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 961 B/s rd, 0 op/s 2024-02-07T12:49:49.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:48 smithi172 ceph-mon[67636]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:49.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:49:49.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:49:49.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:49.377 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:49:50.056 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:49 smithi019 ceph-mon[82918]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:49:50.056 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:49 smithi019 ceph-mon[82918]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:49:50.056 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:49 smithi019 ceph-mon[88137]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:49:50.056 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:49 smithi019 ceph-mon[88137]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:49:50.057 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:49:50.057 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (4m) 76s ago 25m 19.7M - 0.25.0 c8568f914cd2 fec9255f35e7 2024-02-07T12:49:50.057 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (3m) 100s ago 25m 82.9M - 9.4.12 a0ea5e3c46d9 aaffac63bc28 2024-02-07T12:49:50.057 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 unknown 76s ago 25m 63.2M - 3.5 e1d6a67b021e 5113aa77e3f2 2024-02-07T12:49:50.057 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283,8765 running (5m) 100s ago 30m 447M - 19.0.0-1252-g44458cce 79957b372307 5aa0c04fdfd7 2024-02-07T12:49:50.057 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (15m) 76s ago 32m 494M - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:49:50.057 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (115s) 76s ago 32m 41.2M 2048M 19.0.0-1252-g44458cce 79957b372307 8b1b0e21b1c5 2024-02-07T12:49:50.057 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (2m) 100s ago 30m 37.0M 2048M 19.0.0-1252-g44458cce 79957b372307 00f454bbac01 2024-02-07T12:49:50.057 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (87s) 76s ago 30m 23.0M 2048M 19.0.0-1252-g44458cce 79957b372307 6214003f6e08 2024-02-07T12:49:50.057 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (4m) 76s ago 26m 8476k - 1.5.0 0da6a335fe13 c4bdcb1c4d49 2024-02-07T12:49:50.057 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (4m) 100s ago 26m 16.3M - 1.5.0 0da6a335fe13 04ea81f79d85 2024-02-07T12:49:50.057 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (30m) 76s ago 30m 54.7M 2993M 17.2.0 e1d6a67b021e 87f2560a9b15 2024-02-07T12:49:50.057 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (29m) 76s ago 29m 57.2M 2993M 17.2.0 e1d6a67b021e 0f9a190f0e0d 2024-02-07T12:49:50.058 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (29m) 76s ago 29m 53.2M 2993M 17.2.0 e1d6a67b021e 7db94f25a1aa 2024-02-07T12:49:50.058 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (28m) 76s ago 28m 53.5M 2993M 17.2.0 e1d6a67b021e 0b6b674188ed 2024-02-07T12:49:50.058 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (28m) 100s ago 28m 55.7M 3248M 17.2.0 e1d6a67b021e df72cfd69b3b 2024-02-07T12:49:50.058 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (28m) 100s ago 28m 52.4M 3248M 17.2.0 e1d6a67b021e 3bdb22c07537 2024-02-07T12:49:50.058 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (27m) 100s ago 27m 52.1M 3248M 17.2.0 e1d6a67b021e 7859b651e0d9 2024-02-07T12:49:50.058 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (27m) 100s ago 27m 55.5M 3248M 17.2.0 e1d6a67b021e b301e7c75d73 2024-02-07T12:49:50.058 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (4m) 100s ago 26m 61.0M - 2.43.0 a07b618ecd1d c9b607eea1be 2024-02-07T12:49:50.058 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (25m) 76s ago 25m 78.9M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:49:50.058 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (25m) 100s ago 25m 78.3M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:49:50.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:49:50.213+0000 7fb71d671700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:49:50.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:49:50.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:49:50.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:49:50.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:49:50.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:50.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:50.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:49:50.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:49:50.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:49:50.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:49:50.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:49:50.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:49:50.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:49:50.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:49:50.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:49:50.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:49:50.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:49:50.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:49:50.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:50.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:50.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:49:50.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:49:50.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:49:50.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:49:50.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:49:50.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:49:50.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:49:50.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:49:50.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:49:50.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:49:50.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:49:50.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:49:50.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:49:50.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:49:50.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:49:50.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:49:50.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:49:50.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:49:50.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:49:50.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:49:50.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:49:50.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:49:50.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:50.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:50.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:49:50.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:49:50.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:49:50.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:49:50.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:49:50.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:49:50.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:49:50.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:49:50.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:49:50.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:49:50.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:49:50.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:49:50.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:50.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:50.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:49 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:49:50.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:49 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:49:50.476 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:49:50.477 INFO:teuthology.orchestra.run.smithi019.stdout: "mon": { 2024-02-07T12:49:50.477 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 3 2024-02-07T12:49:50.477 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:49:50.477 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr": { 2024-02-07T12:49:50.477 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:49:50.477 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:49:50.477 INFO:teuthology.orchestra.run.smithi019.stdout: "osd": { 2024-02-07T12:49:50.477 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-02-07T12:49:50.477 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:49:50.477 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw": { 2024-02-07T12:49:50.477 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:49:50.477 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:49:50.478 INFO:teuthology.orchestra.run.smithi019.stdout: "overall": { 2024-02-07T12:49:50.478 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-02-07T12:49:50.478 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 5 2024-02-07T12:49:50.478 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:49:50.478 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:49:50.857 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:49:50.857 INFO:teuthology.orchestra.run.smithi019.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", 2024-02-07T12:49:50.857 INFO:teuthology.orchestra.run.smithi019.stdout: "in_progress": true, 2024-02-07T12:49:50.857 INFO:teuthology.orchestra.run.smithi019.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (2 remaining).", 2024-02-07T12:49:50.857 INFO:teuthology.orchestra.run.smithi019.stdout: "services_complete": [], 2024-02-07T12:49:50.857 INFO:teuthology.orchestra.run.smithi019.stdout: "progress": "", 2024-02-07T12:49:50.857 INFO:teuthology.orchestra.run.smithi019.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 image", 2024-02-07T12:49:50.857 INFO:teuthology.orchestra.run.smithi019.stdout: "is_paused": false 2024-02-07T12:49:50.857 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:49:51.213 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-mon[88137]: from='client.54181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:49:51.213 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-mon[88137]: from='client.44250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:49:51.213 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-mon[88137]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 591 B/s rd, 0 op/s 2024-02-07T12:49:51.213 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:49:51.213 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/1624255381' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:51.213 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-mon[82918]: from='client.54181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:49:51.213 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-mon[82918]: from='client.44250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:49:51.213 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-mon[82918]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 591 B/s rd, 0 op/s 2024-02-07T12:49:51.213 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:49:51.213 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:50 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/1624255381' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:50 smithi172 ceph-mon[67636]: from='client.54181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:49:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:50 smithi172 ceph-mon[67636]: from='client.44250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:49:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:50 smithi172 ceph-mon[67636]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 591 B/s rd, 0 op/s 2024-02-07T12:49:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:50 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:49:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:50 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/1624255381' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:52.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:51 smithi019 ceph-mon[88137]: from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:49:52.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:51 smithi019 ceph-mon[88137]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:49:52.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:49:52.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:49:52.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:49:52.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:49:52.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:52.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:52.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:49:52.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:49:52.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:49:52.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:49:52.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:49:52.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:49:52.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:49:52.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:49:52.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:49:52.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:49:52.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:49:52.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:49:52.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:52.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:52.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:49:52.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:49:52.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:49:52.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:49:52.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:49:52.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:49:52.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:49:52.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:49:52.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:49:52.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:49:52.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:49:52.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:49:52.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:49:52.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:49:52.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:49:52.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:49:52.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:49:52.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:49:52.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:49:52.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:49:52.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:49:52.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:49:52.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:52.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:52.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:49:52.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:49:52.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:49:52.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:49:52.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:49:52.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:49:52.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:49:52.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:49:52.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:49:52.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:51 smithi019 ceph-mon[82918]: from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:49:52.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:51 smithi019 ceph-mon[82918]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:49:52.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:49:52.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:49:52.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:49:52.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:49:52.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:52.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:52.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:49:52.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:49:52.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:49:52.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:49:52.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:49:52.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:49:52.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:49:52.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:49:52.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:49:52.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:49:52.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:49:52.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:49:52.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:52.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:52.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:49:52.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:49:52.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:49:52.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:49:52.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:49:52.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:49:52.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:49:52.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:49:52.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:49:52.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:49:52.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:49:52.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:49:52.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:49:52.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:49:52.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:49:52.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:49:52.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:49:52.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:49:52.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:49:52.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:49:52.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:49:52.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:49:52.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:52.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:52.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:49:52.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:49:52.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:49:52.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:49:52.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:49:52.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:49:52.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:49:52.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:49:52.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:49:52.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:49:52.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:49:52.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:49:52.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:52.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:52.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:51 smithi019 ceph-mon[88137]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:49:52.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:51 smithi019 ceph-mon[88137]: from='client.54205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:49:52.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:49:52.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:49:52.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:49:52.354 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:52.354 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:52.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:51 smithi019 ceph-mon[82918]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:49:52.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:51 smithi019 ceph-mon[82918]: from='client.54205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:49:52.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:51 smithi172 ceph-mon[67636]: from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:49:52.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:51 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:49:52.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:49:52.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:49:52.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:49:52.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:49:52.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:52.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:52.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:49:52.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:49:52.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:49:52.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:49:52.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:49:52.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:49:52.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:49:52.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:49:52.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:49:52.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:49:52.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:49:52.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:49:52.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:52.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:52.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:49:52.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:49:52.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:49:52.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:49:52.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:49:52.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:49:52.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:49:52.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:49:52.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:49:52.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:49:52.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:49:52.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:49:52.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:49:52.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:49:52.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:49:52.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:49:52.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:49:52.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:49:52.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:49:52.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:49:52.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:49:52.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:49:52.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:52.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:52.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:49:52.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:49:52.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:49:52.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:49:52.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:49:52.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:49:52.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:49:52.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:49:52.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:49:52.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:49:52.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:49:52.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:49:52.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:49:52.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:49:52.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:51 smithi172 ceph-mon[67636]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:49:52.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:51 smithi172 ceph-mon[67636]: from='client.54205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:49:53.270 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:52 smithi019 ceph-mon[88137]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:53.270 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:52 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:53.270 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:52 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:49:53.270 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:52 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:53.271 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:52 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:53.271 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:52 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:53.271 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:52 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:53.271 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:52 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:53.271 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:52 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:53.271 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:52 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-02-07T12:49:53.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:52 smithi019 ceph-mon[82918]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:53.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:52 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:53.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:52 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:49:53.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:52 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:53.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:52 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:53.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:52 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:53.272 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:52 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:53.272 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:52 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:53.272 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:52 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:53.272 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:52 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-02-07T12:49:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:52 smithi172 ceph-mon[67636]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:52 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:52 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:49:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:52 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:52 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:53.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:52 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:53.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:52 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:53.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:52 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:49:53.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:52 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:53.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:52 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-02-07T12:49:53.966 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:49:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:49:53] "GET /metrics HTTP/1.1" 200 38006 "" "Prometheus/2.43.0" 2024-02-07T12:49:54.251 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:53 smithi019 ceph-mon[88137]: Upgrade: Target is version 19.0.0-1252-g44458cce (squid) 2024-02-07T12:49:54.251 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:53 smithi019 ceph-mon[88137]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe'] 2024-02-07T12:49:54.251 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:53 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all mgr 2024-02-07T12:49:54.251 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:53 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all mon 2024-02-07T12:49:54.251 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:53 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all crash 2024-02-07T12:49:54.251 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:53 smithi019 ceph-mon[88137]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-02-07T12:49:54.251 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:53 smithi019 ceph-mon[88137]: Upgrade: osd.0 is safe to restart 2024-02-07T12:49:54.251 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:53 smithi019 ceph-mon[82918]: Upgrade: Target is version 19.0.0-1252-g44458cce (squid) 2024-02-07T12:49:54.251 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:53 smithi019 ceph-mon[82918]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe'] 2024-02-07T12:49:54.251 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:53 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all mgr 2024-02-07T12:49:54.252 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:53 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all mon 2024-02-07T12:49:54.252 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:53 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all crash 2024-02-07T12:49:54.252 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:53 smithi019 ceph-mon[82918]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-02-07T12:49:54.252 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:53 smithi019 ceph-mon[82918]: Upgrade: osd.0 is safe to restart 2024-02-07T12:49:54.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:53 smithi172 ceph-mon[67636]: Upgrade: Target is version 19.0.0-1252-g44458cce (squid) 2024-02-07T12:49:54.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:53 smithi172 ceph-mon[67636]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe'] 2024-02-07T12:49:54.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:53 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all mgr 2024-02-07T12:49:54.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:53 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all mon 2024-02-07T12:49:54.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:53 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all crash 2024-02-07T12:49:54.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:53 smithi172 ceph-mon[67636]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-02-07T12:49:54.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:53 smithi172 ceph-mon[67636]: Upgrade: osd.0 is safe to restart 2024-02-07T12:49:55.187 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:54 smithi019 ceph-mon[82918]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:55.187 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:54 smithi019 ceph-mon[82918]: Upgrade: Updating osd.0 2024-02-07T12:49:55.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:54 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:55.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:54 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-02-07T12:49:55.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:54 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:55.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:54 smithi019 ceph-mon[82918]: Deploying daemon osd.0 on smithi019 2024-02-07T12:49:55.188 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:54 smithi019 ceph-mon[88137]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:55.188 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:54 smithi019 ceph-mon[88137]: Upgrade: Updating osd.0 2024-02-07T12:49:55.188 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:54 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:55.188 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:54 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-02-07T12:49:55.188 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:54 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:55.188 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:54 smithi019 ceph-mon[88137]: Deploying daemon osd.0 on smithi019 2024-02-07T12:49:55.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:54 smithi172 ceph-mon[67636]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:49:55.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:54 smithi172 ceph-mon[67636]: Upgrade: Updating osd.0 2024-02-07T12:49:55.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:54 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:49:55.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:54 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-02-07T12:49:55.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:54 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:49:55.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:54 smithi172 ceph-mon[67636]: Deploying daemon osd.0 on smithi019 2024-02-07T12:49:56.117 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:55 smithi019 ceph-mon[82918]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 591 B/s rd, 0 op/s 2024-02-07T12:49:56.118 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:55 smithi019 ceph-mon[88137]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 591 B/s rd, 0 op/s 2024-02-07T12:49:56.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:55 smithi172 ceph-mon[67636]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 591 B/s rd, 0 op/s 2024-02-07T12:49:57.241 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:56 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:49:57.241 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:56 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:49:57.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:56 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:49:57.872 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:49:57 smithi019 systemd[1]: Stopping Ceph osd.0 for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:49:58.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:57 smithi019 ceph-mon[82918]: osd.0 marked itself down and dead 2024-02-07T12:49:58.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:57 smithi019 ceph-mon[82918]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:49:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:57 smithi019 ceph-mon[88137]: osd.0 marked itself down and dead 2024-02-07T12:49:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:57 smithi019 ceph-mon[88137]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:49:58.345 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:49:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0[32130]: 2024-02-07T12:49:57.869+0000 7f68fcfa4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:49:58.345 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:49:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0[32130]: 2024-02-07T12:49:57.869+0000 7f68fcfa4700 -1 osd.0 105 *** Got signal Terminated *** 2024-02-07T12:49:58.345 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:49:57 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0[32130]: 2024-02-07T12:49:57.869+0000 7f68fcfa4700 -1 osd.0 105 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:49:58.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:57 smithi172 ceph-mon[67636]: osd.0 marked itself down and dead 2024-02-07T12:49:58.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:57 smithi172 ceph-mon[67636]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:49:58.713 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:49:58 smithi019 podman[95622]: 2024-02-07 12:49:58.441292407 +0000 UTC m=+0.663670746 container died 87f2560a9b15ff1c3c979a2b6a4c9eee7c903e6f94861f0e2799ac3aef9ea8b4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., version=8, maintainer=Guillaume Abrioux , RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, release=754, name=centos-stream, vcs-type=git, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-02-07T12:49:58.987 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:49:58 smithi019 podman[95622]: 2024-02-07 12:49:58.794275659 +0000 UTC m=+1.016653998 container cleanup 87f2560a9b15ff1c3c979a2b6a4c9eee7c903e6f94861f0e2799ac3aef9ea8b4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, release=754, vcs-type=git, io.openshift.tags=base centos centos-stream, distribution-scope=public, RELEASE=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-02-07T12:49:58.988 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:49:58 smithi019 bash[95622]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0 2024-02-07T12:49:59.344 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:49:59 smithi019 podman[95649]: 2024-02-07 12:49:59.087617414 +0000 UTC m=+0.648336729 container remove 87f2560a9b15ff1c3c979a2b6a4c9eee7c903e6f94861f0e2799ac3aef9ea8b4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0, RELEASE=HEAD, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, ceph=True, version=8, com.redhat.component=centos-stream-container) 2024-02-07T12:49:59.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:58 smithi019 ceph-mon[82918]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:49:59.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:58 smithi019 ceph-mon[82918]: failed to encode map e106 with expected crc 2024-02-07T12:49:59.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:58 smithi019 ceph-mon[82918]: failed to encode map e106 with expected crc 2024-02-07T12:49:59.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:58 smithi019 ceph-mon[82918]: osdmap e106: 8 total, 7 up, 8 in 2024-02-07T12:49:59.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:58 smithi019 ceph-mon[82918]: failed to encode map e106 with expected crc 2024-02-07T12:49:59.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:58 smithi019 ceph-mon[82918]: failed to encode map e106 with expected crc 2024-02-07T12:49:59.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:58 smithi019 ceph-mon[88137]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:49:59.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:58 smithi019 ceph-mon[88137]: failed to encode map e106 with expected crc 2024-02-07T12:49:59.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:58 smithi019 ceph-mon[88137]: failed to encode map e106 with expected crc 2024-02-07T12:49:59.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:58 smithi019 ceph-mon[88137]: osdmap e106: 8 total, 7 up, 8 in 2024-02-07T12:49:59.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:58 smithi019 ceph-mon[88137]: failed to encode map e106 with expected crc 2024-02-07T12:49:59.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:58 smithi019 ceph-mon[88137]: failed to encode map e106 with expected crc 2024-02-07T12:49:59.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:58 smithi172 ceph-mon[67636]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:49:59.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:58 smithi172 ceph-mon[67636]: failed to encode map e106 with expected crc 2024-02-07T12:49:59.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:58 smithi172 ceph-mon[67636]: failed to encode map e106 with expected crc 2024-02-07T12:49:59.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:58 smithi172 ceph-mon[67636]: osdmap e106: 8 total, 7 up, 8 in 2024-02-07T12:49:59.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:58 smithi172 ceph-mon[67636]: failed to encode map e106 with expected crc 2024-02-07T12:49:59.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:58 smithi172 ceph-mon[67636]: failed to encode map e106 with expected crc 2024-02-07T12:49:59.751 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:49:59 smithi019 podman[95746]: 2024-02-07 12:49:59.314168195 +0000 UTC m=+0.018983378 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:49:59.751 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:49:59 smithi019 podman[95746]: 2024-02-07 12:49:59.421871661 +0000 UTC m=+0.126686843 container create c3b42bcbe2ebd9e81736de667ab07879227bc0c27ae7b399356331e0ec2bfa9f (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0-deactivate, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240131, ceph=True, GIT_CLEAN=True) 2024-02-07T12:49:59.752 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:49:59 smithi019 podman[95746]: 2024-02-07 12:49:59.649953736 +0000 UTC m=+0.354768935 container init c3b42bcbe2ebd9e81736de667ab07879227bc0c27ae7b399356331e0ec2bfa9f (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , io.buildah.version=1.33.2) 2024-02-07T12:49:59.752 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:49:59 smithi019 podman[95746]: 2024-02-07 12:49:59.653337593 +0000 UTC m=+0.358152784 container start c3b42bcbe2ebd9e81736de667ab07879227bc0c27ae7b399356331e0ec2bfa9f (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.2) 2024-02-07T12:49:59.752 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:49:59 smithi019 podman[95746]: 2024-02-07 12:49:59.749971215 +0000 UTC m=+0.454786415 container attach c3b42bcbe2ebd9e81736de667ab07879227bc0c27ae7b399356331e0ec2bfa9f (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-02-07T12:50:00.094 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:49:59 smithi019 podman[95746]: 2024-02-07 12:49:59.961114043 +0000 UTC m=+0.665929226 container died c3b42bcbe2ebd9e81736de667ab07879227bc0c27ae7b399356331e0ec2bfa9f (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:50:00.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:59 smithi019 ceph-mon[88137]: failed to encode map e107 with expected crc 2024-02-07T12:50:00.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:59 smithi019 ceph-mon[88137]: failed to encode map e107 with expected crc 2024-02-07T12:50:00.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:59 smithi019 ceph-mon[88137]: osdmap e107: 8 total, 7 up, 8 in 2024-02-07T12:50:00.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:59 smithi019 ceph-mon[88137]: failed to encode map e107 with expected crc 2024-02-07T12:50:00.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:59 smithi019 ceph-mon[88137]: failed to encode map e107 with expected crc 2024-02-07T12:50:00.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:59 smithi019 ceph-mon[88137]: failed to encode map e107 with expected crc 2024-02-07T12:50:00.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:59 smithi019 ceph-mon[88137]: failed to encode map e107 with expected crc 2024-02-07T12:50:00.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:59 smithi019 ceph-mon[88137]: failed to encode map e107 with expected crc 2024-02-07T12:50:00.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:59 smithi019 ceph-mon[88137]: failed to encode map e107 with expected crc 2024-02-07T12:50:00.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:49:59 smithi019 ceph-mon[88137]: pgmap v70: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-02-07T12:50:00.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:59 smithi019 ceph-mon[82918]: failed to encode map e107 with expected crc 2024-02-07T12:50:00.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:59 smithi019 ceph-mon[82918]: failed to encode map e107 with expected crc 2024-02-07T12:50:00.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:59 smithi019 ceph-mon[82918]: osdmap e107: 8 total, 7 up, 8 in 2024-02-07T12:50:00.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:59 smithi019 ceph-mon[82918]: failed to encode map e107 with expected crc 2024-02-07T12:50:00.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:59 smithi019 ceph-mon[82918]: failed to encode map e107 with expected crc 2024-02-07T12:50:00.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:59 smithi019 ceph-mon[82918]: failed to encode map e107 with expected crc 2024-02-07T12:50:00.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:59 smithi019 ceph-mon[82918]: failed to encode map e107 with expected crc 2024-02-07T12:50:00.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:59 smithi019 ceph-mon[82918]: failed to encode map e107 with expected crc 2024-02-07T12:50:00.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:59 smithi019 ceph-mon[82918]: failed to encode map e107 with expected crc 2024-02-07T12:50:00.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:49:59 smithi019 ceph-mon[82918]: pgmap v70: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-02-07T12:50:00.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:59 smithi172 ceph-mon[67636]: failed to encode map e107 with expected crc 2024-02-07T12:50:00.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:59 smithi172 ceph-mon[67636]: failed to encode map e107 with expected crc 2024-02-07T12:50:00.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:59 smithi172 ceph-mon[67636]: osdmap e107: 8 total, 7 up, 8 in 2024-02-07T12:50:00.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:59 smithi172 ceph-mon[67636]: failed to encode map e107 with expected crc 2024-02-07T12:50:00.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:59 smithi172 ceph-mon[67636]: failed to encode map e107 with expected crc 2024-02-07T12:50:00.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:59 smithi172 ceph-mon[67636]: failed to encode map e107 with expected crc 2024-02-07T12:50:00.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:59 smithi172 ceph-mon[67636]: failed to encode map e107 with expected crc 2024-02-07T12:50:00.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:59 smithi172 ceph-mon[67636]: failed to encode map e107 with expected crc 2024-02-07T12:50:00.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:59 smithi172 ceph-mon[67636]: failed to encode map e107 with expected crc 2024-02-07T12:50:00.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:49:59 smithi172 ceph-mon[67636]: pgmap v70: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-02-07T12:50:00.844 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:00 smithi019 podman[95773]: 2024-02-07 12:50:00.487118437 +0000 UTC m=+0.516693231 container remove c3b42bcbe2ebd9e81736de667ab07879227bc0c27ae7b399356331e0ec2bfa9f (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:50:00.844 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:00 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.0.service: Deactivated successfully. 2024-02-07T12:50:00.844 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:00 smithi019 systemd[1]: Stopped Ceph osd.0 for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:50:00.844 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:00 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.0.service: Consumed 14.190s CPU time. 2024-02-07T12:50:00.844 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:00 smithi019 systemd[1]: Starting Ceph osd.0 for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:50:01.243 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:00 smithi019 ceph-mon[88137]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down 2024-02-07T12:50:01.243 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:00 smithi019 ceph-mon[88137]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-02-07T12:50:01.243 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:00 smithi019 ceph-mon[88137]: daemon iscsi.foo.smithi019.svjimj on smithi019 is in unknown state 2024-02-07T12:50:01.243 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:00 smithi019 ceph-mon[88137]: [WRN] OSD_DOWN: 1 osds down 2024-02-07T12:50:01.243 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:00 smithi019 ceph-mon[88137]: osd.0 (root=default,host=smithi019) is down 2024-02-07T12:50:01.243 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:00 smithi019 ceph-mon[82918]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down 2024-02-07T12:50:01.243 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:00 smithi019 ceph-mon[82918]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-02-07T12:50:01.243 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:00 smithi019 ceph-mon[82918]: daemon iscsi.foo.smithi019.svjimj on smithi019 is in unknown state 2024-02-07T12:50:01.244 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:00 smithi019 ceph-mon[82918]: [WRN] OSD_DOWN: 1 osds down 2024-02-07T12:50:01.244 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:00 smithi019 ceph-mon[82918]: osd.0 (root=default,host=smithi019) is down 2024-02-07T12:50:01.244 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:00 smithi019 podman[95899]: 2024-02-07 12:50:00.835391948 +0000 UTC m=+0.017588242 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:50:01.244 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:00 smithi019 podman[95899]: 2024-02-07 12:50:00.946731324 +0000 UTC m=+0.128927615 container create 11914acc82530d9d94c2b6529d24f3e8a980dca8d2067d891fc65dc0d4e91b57 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0-activate, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240131, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:50:01.244 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:01 smithi019 podman[95899]: 2024-02-07 12:50:01.162043997 +0000 UTC m=+0.344240288 container init 11914acc82530d9d94c2b6529d24f3e8a980dca8d2067d891fc65dc0d4e91b57 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0-activate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-02-07T12:50:01.244 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:01 smithi019 podman[95899]: 2024-02-07 12:50:01.165079842 +0000 UTC m=+0.347276138 container start 11914acc82530d9d94c2b6529d24f3e8a980dca8d2067d891fc65dc0d4e91b57 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0-activate, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True) 2024-02-07T12:50:01.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:00 smithi172 ceph-mon[67636]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down 2024-02-07T12:50:01.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:00 smithi172 ceph-mon[67636]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-02-07T12:50:01.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:00 smithi172 ceph-mon[67636]: daemon iscsi.foo.smithi019.svjimj on smithi019 is in unknown state 2024-02-07T12:50:01.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:00 smithi172 ceph-mon[67636]: [WRN] OSD_DOWN: 1 osds down 2024-02-07T12:50:01.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:00 smithi172 ceph-mon[67636]: osd.0 (root=default,host=smithi019) is down 2024-02-07T12:50:01.594 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:01 smithi019 podman[95899]: 2024-02-07 12:50:01.241177325 +0000 UTC m=+0.423373617 container attach 11914acc82530d9d94c2b6529d24f3e8a980dca8d2067d891fc65dc0d4e91b57 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0-activate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-02-07T12:50:01.946 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:01 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0-activate[95913]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-02-07T12:50:01.946 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:01 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0-activate[95913]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-02-07T12:50:01.947 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:01 smithi019 bash[95899]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-02-07T12:50:01.947 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:01 smithi019 bash[95899]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-02-07T12:50:01.947 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:01 smithi019 bash[95899]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-02-07T12:50:01.947 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:01 smithi019 bash[95899]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-02-07T12:50:01.947 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:01 smithi019 bash[95899]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-02-07T12:50:01.947 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:01 smithi019 bash[95899]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-02-07T12:50:01.947 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:01 smithi019 bash[95899]: --> ceph-volume raw activate successful for osd ID: 0 2024-02-07T12:50:01.947 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:01 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0-activate[95913]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-02-07T12:50:01.947 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:01 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0-activate[95913]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-02-07T12:50:01.947 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:01 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0-activate[95913]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-02-07T12:50:01.947 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:01 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0-activate[95913]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-02-07T12:50:01.947 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:01 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0-activate[95913]: --> ceph-volume raw activate successful for osd ID: 0 2024-02-07T12:50:01.947 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:01 smithi019 podman[95899]: 2024-02-07 12:50:01.639692602 +0000 UTC m=+0.821888899 container died 11914acc82530d9d94c2b6529d24f3e8a980dca8d2067d891fc65dc0d4e91b57 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0-activate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD) 2024-02-07T12:50:02.214 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:02 smithi019 ceph-mon[82918]: pgmap v71: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 67/627 objects degraded (10.686%) 2024-02-07T12:50:02.214 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:02 smithi019 ceph-mon[88137]: pgmap v71: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 67/627 objects degraded (10.686%) 2024-02-07T12:50:02.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:02 smithi172 ceph-mon[67636]: pgmap v71: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 67/627 objects degraded (10.686%) 2024-02-07T12:50:02.521 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:02 smithi019 podman[96047]: 2024-02-07 12:50:02.212242515 +0000 UTC m=+0.567568137 container remove 11914acc82530d9d94c2b6529d24f3e8a980dca8d2067d891fc65dc0d4e91b57 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0-activate, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-02-07T12:50:02.521 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:02 smithi019 podman[96110]: 2024-02-07 12:50:02.421504209 +0000 UTC m=+0.112634315 container create 78eddd34a903d9964edb15fe097be997606a173da04078f886cfcf6414cf48db (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-02-07T12:50:02.521 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:02 smithi019 podman[96110]: 2024-02-07 12:50:02.326493515 +0000 UTC m=+0.017623570 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:50:03.008 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:02 smithi019 podman[96110]: 2024-02-07 12:50:02.615548732 +0000 UTC m=+0.306678787 container init 78eddd34a903d9964edb15fe097be997606a173da04078f886cfcf6414cf48db (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0, org.label-schema.build-date=20240131, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-02-07T12:50:03.008 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:02 smithi019 podman[96110]: 2024-02-07 12:50:02.618520453 +0000 UTC m=+0.309650511 container start 78eddd34a903d9964edb15fe097be997606a173da04078f886cfcf6414cf48db (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.2, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:50:03.008 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:02 smithi019 ceph-osd[96129]: -- 172.21.15.19:0/498791400 <== mon.1 v2:172.21.15.19:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55bdabbf96c0 con 0x55bdabbcac00 2024-02-07T12:50:03.009 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:02 smithi019 bash[96110]: 78eddd34a903d9964edb15fe097be997606a173da04078f886cfcf6414cf48db 2024-02-07T12:50:03.009 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:02 smithi019 systemd[1]: Started Ceph osd.0 for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:50:03.344 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0[96125]: 2024-02-07T12:50:03.158+0000 7f6bc8f3f7c0 -1 Falling back to public interface 2024-02-07T12:50:03.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:03 smithi019 ceph-mon[82918]: Health check failed: Degraded data redundancy: 67/627 objects degraded (10.686%), 21 pgs degraded (PG_DEGRADED) 2024-02-07T12:50:03.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:03 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:03.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:03 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:03.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:03 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:03.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:03 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:50:03.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:03 smithi019 ceph-mon[88137]: Health check failed: Degraded data redundancy: 67/627 objects degraded (10.686%), 21 pgs degraded (PG_DEGRADED) 2024-02-07T12:50:03.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:03 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:03.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:03 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:03.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:03 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:03.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:03 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:50:03.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:03 smithi172 ceph-mon[67636]: Health check failed: Degraded data redundancy: 67/627 objects degraded (10.686%), 21 pgs degraded (PG_DEGRADED) 2024-02-07T12:50:03.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:03 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:03.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:03 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:03.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:03 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:03.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:03 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:50:04.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:04 smithi019 ceph-mon[82918]: pgmap v72: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 67/627 objects degraded (10.686%) 2024-02-07T12:50:04.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:04 smithi019 ceph-mon[88137]: pgmap v72: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 67/627 objects degraded (10.686%) 2024-02-07T12:50:04.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:50:03] "GET /metrics HTTP/1.1" 200 38006 "" "Prometheus/2.43.0" 2024-02-07T12:50:04.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:04 smithi172 ceph-mon[67636]: pgmap v72: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 67/627 objects degraded (10.686%) 2024-02-07T12:50:06.235 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:06 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:06.236 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:06 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:50:06.236 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:06 smithi019 ceph-mon[88137]: pgmap v73: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 67/627 objects degraded (10.686%) 2024-02-07T12:50:06.236 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:06 smithi019 ceph-mon[88137]: from='osd.0 [v2:172.21.15.19:6802/1513253987,v1:172.21.15.19:6803/1513253987]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-02-07T12:50:06.236 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:06 smithi019 ceph-mon[88137]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-02-07T12:50:06.236 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:05 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0[96125]: 2024-02-07T12:50:05.863+0000 7f6bc8f3f7c0 -1 osd.0 0 read_superblock omap replica is missing. 2024-02-07T12:50:06.236 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:05 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0[96125]: 2024-02-07T12:50:05.896+0000 7f6bc8f3f7c0 -1 osd.0 105 log_to_monitors true 2024-02-07T12:50:06.236 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:06 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:06.237 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:06 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:50:06.237 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:06 smithi019 ceph-mon[82918]: pgmap v73: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 67/627 objects degraded (10.686%) 2024-02-07T12:50:06.237 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:06 smithi019 ceph-mon[82918]: from='osd.0 [v2:172.21.15.19:6802/1513253987,v1:172.21.15.19:6803/1513253987]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-02-07T12:50:06.237 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:06 smithi019 ceph-mon[82918]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-02-07T12:50:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:06 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:06 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:50:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:06 smithi172 ceph-mon[67636]: pgmap v73: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 67/627 objects degraded (10.686%) 2024-02-07T12:50:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:06 smithi172 ceph-mon[67636]: from='osd.0 [v2:172.21.15.19:6802/1513253987,v1:172.21.15.19:6803/1513253987]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-02-07T12:50:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:06 smithi172 ceph-mon[67636]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-02-07T12:50:06.844 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:50:06 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0[96125]: 2024-02-07T12:50:06.581+0000 7f6bb6396700 -1 osd.0 105 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:50:07.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:07 smithi019 ceph-mon[88137]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-02-07T12:50:07.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:07 smithi019 ceph-mon[88137]: failed to encode map e108 with expected crc 2024-02-07T12:50:07.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:07 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:50:07.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:07 smithi019 ceph-mon[88137]: osdmap e108: 8 total, 7 up, 8 in 2024-02-07T12:50:07.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:07 smithi019 ceph-mon[88137]: from='osd.0 [v2:172.21.15.19:6802/1513253987,v1:172.21.15.19:6803/1513253987]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:50:07.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:07 smithi019 ceph-mon[88137]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:50:07.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:07 smithi019 ceph-mon[88137]: from='osd.0 ' entity='osd.0' 2024-02-07T12:50:07.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:07 smithi019 ceph-mon[82918]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-02-07T12:50:07.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:07 smithi019 ceph-mon[82918]: failed to encode map e108 with expected crc 2024-02-07T12:50:07.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:07 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:50:07.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:07 smithi019 ceph-mon[82918]: osdmap e108: 8 total, 7 up, 8 in 2024-02-07T12:50:07.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:07 smithi019 ceph-mon[82918]: from='osd.0 [v2:172.21.15.19:6802/1513253987,v1:172.21.15.19:6803/1513253987]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:50:07.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:07 smithi019 ceph-mon[82918]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:50:07.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:07 smithi019 ceph-mon[82918]: from='osd.0 ' entity='osd.0' 2024-02-07T12:50:07.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:07 smithi172 ceph-mon[67636]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-02-07T12:50:07.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:07 smithi172 ceph-mon[67636]: failed to encode map e108 with expected crc 2024-02-07T12:50:07.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:07 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:50:07.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:07 smithi172 ceph-mon[67636]: osdmap e108: 8 total, 7 up, 8 in 2024-02-07T12:50:07.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:07 smithi172 ceph-mon[67636]: from='osd.0 [v2:172.21.15.19:6802/1513253987,v1:172.21.15.19:6803/1513253987]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:50:07.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:07 smithi172 ceph-mon[67636]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:50:07.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:07 smithi172 ceph-mon[67636]: from='osd.0 ' entity='osd.0' 2024-02-07T12:50:08.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:08 smithi019 ceph-mon[82918]: failed to encode map e108 with expected crc 2024-02-07T12:50:08.358 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:08 smithi019 ceph-mon[82918]: failed to encode map e108 with expected crc 2024-02-07T12:50:08.359 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:08 smithi019 ceph-mon[82918]: failed to encode map e108 with expected crc 2024-02-07T12:50:08.359 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:08 smithi019 ceph-mon[82918]: failed to encode map e108 with expected crc 2024-02-07T12:50:08.359 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:08 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:08.359 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:08 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:08.359 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:08 smithi019 ceph-mon[82918]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:50:08.359 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:08 smithi019 ceph-mon[82918]: osd.0 [v2:172.21.15.19:6802/1513253987,v1:172.21.15.19:6803/1513253987] boot 2024-02-07T12:50:08.359 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:08 smithi019 ceph-mon[82918]: osdmap e109: 8 total, 8 up, 8 in 2024-02-07T12:50:08.359 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:08 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:50:08.359 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:08 smithi019 ceph-mon[82918]: pgmap v76: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 113 MiB used, 715 GiB / 715 GiB avail; 67/627 objects degraded (10.686%) 2024-02-07T12:50:08.359 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:08 smithi019 ceph-mon[88137]: failed to encode map e108 with expected crc 2024-02-07T12:50:08.359 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:08 smithi019 ceph-mon[88137]: failed to encode map e108 with expected crc 2024-02-07T12:50:08.360 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:08 smithi019 ceph-mon[88137]: failed to encode map e108 with expected crc 2024-02-07T12:50:08.360 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:08 smithi019 ceph-mon[88137]: failed to encode map e108 with expected crc 2024-02-07T12:50:08.360 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:08 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:08.360 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:08 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:08.360 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:08 smithi019 ceph-mon[88137]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:50:08.360 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:08 smithi019 ceph-mon[88137]: osd.0 [v2:172.21.15.19:6802/1513253987,v1:172.21.15.19:6803/1513253987] boot 2024-02-07T12:50:08.360 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:08 smithi019 ceph-mon[88137]: osdmap e109: 8 total, 8 up, 8 in 2024-02-07T12:50:08.360 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:08 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:50:08.360 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:08 smithi019 ceph-mon[88137]: pgmap v76: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 113 MiB used, 715 GiB / 715 GiB avail; 67/627 objects degraded (10.686%) 2024-02-07T12:50:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:08 smithi172 ceph-mon[67636]: failed to encode map e108 with expected crc 2024-02-07T12:50:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:08 smithi172 ceph-mon[67636]: failed to encode map e108 with expected crc 2024-02-07T12:50:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:08 smithi172 ceph-mon[67636]: failed to encode map e108 with expected crc 2024-02-07T12:50:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:08 smithi172 ceph-mon[67636]: failed to encode map e108 with expected crc 2024-02-07T12:50:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:08 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:08 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:08 smithi172 ceph-mon[67636]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:50:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:08 smithi172 ceph-mon[67636]: osd.0 [v2:172.21.15.19:6802/1513253987,v1:172.21.15.19:6803/1513253987] boot 2024-02-07T12:50:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:08 smithi172 ceph-mon[67636]: osdmap e109: 8 total, 8 up, 8 in 2024-02-07T12:50:08.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:08 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:50:08.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:08 smithi172 ceph-mon[67636]: pgmap v76: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 113 MiB used, 715 GiB / 715 GiB avail; 67/627 objects degraded (10.686%) 2024-02-07T12:50:09.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:09 smithi019 ceph-mon[88137]: failed to encode map e109 with expected crc 2024-02-07T12:50:09.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:09 smithi019 ceph-mon[88137]: failed to encode map e109 with expected crc 2024-02-07T12:50:09.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:09 smithi019 ceph-mon[88137]: failed to encode map e109 with expected crc 2024-02-07T12:50:09.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:09 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:09.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:09 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:09.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:09 smithi019 ceph-mon[88137]: failed to encode map e110 with expected crc 2024-02-07T12:50:09.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:09 smithi019 ceph-mon[88137]: failed to encode map e110 with expected crc 2024-02-07T12:50:09.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:09 smithi019 ceph-mon[88137]: osdmap e110: 8 total, 8 up, 8 in 2024-02-07T12:50:09.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:09 smithi019 ceph-mon[88137]: failed to encode map e110 with expected crc 2024-02-07T12:50:09.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:09 smithi019 ceph-mon[88137]: failed to encode map e110 with expected crc 2024-02-07T12:50:09.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:09 smithi019 ceph-mon[88137]: failed to encode map e110 with expected crc 2024-02-07T12:50:09.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:09 smithi019 ceph-mon[82918]: failed to encode map e109 with expected crc 2024-02-07T12:50:09.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:09 smithi019 ceph-mon[82918]: failed to encode map e109 with expected crc 2024-02-07T12:50:09.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:09 smithi019 ceph-mon[82918]: failed to encode map e109 with expected crc 2024-02-07T12:50:09.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:09 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:09.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:09 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:09.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:09 smithi019 ceph-mon[82918]: failed to encode map e110 with expected crc 2024-02-07T12:50:09.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:09 smithi019 ceph-mon[82918]: failed to encode map e110 with expected crc 2024-02-07T12:50:09.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:09 smithi019 ceph-mon[82918]: osdmap e110: 8 total, 8 up, 8 in 2024-02-07T12:50:09.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:09 smithi019 ceph-mon[82918]: failed to encode map e110 with expected crc 2024-02-07T12:50:09.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:09 smithi019 ceph-mon[82918]: failed to encode map e110 with expected crc 2024-02-07T12:50:09.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:09 smithi019 ceph-mon[82918]: failed to encode map e110 with expected crc 2024-02-07T12:50:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:09 smithi172 ceph-mon[67636]: failed to encode map e109 with expected crc 2024-02-07T12:50:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:09 smithi172 ceph-mon[67636]: failed to encode map e109 with expected crc 2024-02-07T12:50:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:09 smithi172 ceph-mon[67636]: failed to encode map e109 with expected crc 2024-02-07T12:50:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:09 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:09 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:09 smithi172 ceph-mon[67636]: failed to encode map e110 with expected crc 2024-02-07T12:50:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:09 smithi172 ceph-mon[67636]: failed to encode map e110 with expected crc 2024-02-07T12:50:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:09 smithi172 ceph-mon[67636]: osdmap e110: 8 total, 8 up, 8 in 2024-02-07T12:50:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:09 smithi172 ceph-mon[67636]: failed to encode map e110 with expected crc 2024-02-07T12:50:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:09 smithi172 ceph-mon[67636]: failed to encode map e110 with expected crc 2024-02-07T12:50:09.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:09 smithi172 ceph-mon[67636]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.288 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:10 smithi019 ceph-mon[82918]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.288 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:10 smithi019 ceph-mon[82918]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.288 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:10 smithi019 ceph-mon[82918]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.288 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:10 smithi019 ceph-mon[82918]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.288 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:10 smithi019 ceph-mon[82918]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.288 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:10 smithi019 ceph-mon[82918]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.289 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:10 smithi019 ceph-mon[82918]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.289 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:10 smithi019 ceph-mon[82918]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.289 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:10 smithi019 ceph-mon[82918]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.289 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:10 smithi019 ceph-mon[82918]: pgmap v78: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 113 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 67/627 objects degraded (10.686%) 2024-02-07T12:50:10.289 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:10 smithi019 ceph-mon[88137]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.289 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:10 smithi019 ceph-mon[88137]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.289 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:10 smithi019 ceph-mon[88137]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.289 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:10 smithi019 ceph-mon[88137]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.289 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:10 smithi019 ceph-mon[88137]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.289 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:10 smithi019 ceph-mon[88137]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.289 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:10 smithi019 ceph-mon[88137]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.289 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:10 smithi019 ceph-mon[88137]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.290 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:10 smithi019 ceph-mon[88137]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.290 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:10 smithi019 ceph-mon[88137]: pgmap v78: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 113 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 67/627 objects degraded (10.686%) 2024-02-07T12:50:10.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:10 smithi172 ceph-mon[67636]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:10 smithi172 ceph-mon[67636]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:10 smithi172 ceph-mon[67636]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:10 smithi172 ceph-mon[67636]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:10 smithi172 ceph-mon[67636]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:10 smithi172 ceph-mon[67636]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:10 smithi172 ceph-mon[67636]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:10 smithi172 ceph-mon[67636]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:10 smithi172 ceph-mon[67636]: failed to encode map e110 with expected crc 2024-02-07T12:50:10.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:10 smithi172 ceph-mon[67636]: pgmap v78: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 113 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 67/627 objects degraded (10.686%) 2024-02-07T12:50:13.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:12 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:13.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:12 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:13.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:12 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:50:13.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:12 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:50:13.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:12 smithi019 ceph-mon[88137]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-02-07T12:50:13.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:12 smithi019 ceph-mon[88137]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 900 B/s rd, 0 op/s 2024-02-07T12:50:13.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:12 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:13.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:12 smithi019 ceph-mon[88137]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:50:13.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:12 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:50:13.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:12 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:50:13.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:12 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:50:13.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:12 smithi019 ceph-mon[88137]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:50:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:12 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:12 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:12 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:50:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:12 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:50:13.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:12 smithi019 ceph-mon[82918]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-02-07T12:50:13.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:12 smithi019 ceph-mon[82918]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 900 B/s rd, 0 op/s 2024-02-07T12:50:13.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:12 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:13.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:12 smithi019 ceph-mon[82918]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:50:13.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:12 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:50:13.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:12 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:50:13.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:12 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:50:13.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:12 smithi019 ceph-mon[82918]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:50:13.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:12 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:13.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:12 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:13.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:12 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:50:13.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:12 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:50:13.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:12 smithi172 ceph-mon[67636]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-02-07T12:50:13.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:12 smithi172 ceph-mon[67636]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 900 B/s rd, 0 op/s 2024-02-07T12:50:13.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:12 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:13.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:12 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:50:13.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:12 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:50:13.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:12 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:50:13.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:12 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:50:13.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:12 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:50:13.758 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:50:13.523+0000 7fb71d671700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:13.758 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:50:13.758 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:13.758 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:13.758 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:13.759 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:13.759 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:13.759 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:50:13.759 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:13.759 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:50:13.759 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:13.759 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:50:13.759 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:13.759 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:13.759 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:13.759 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:13.759 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:13.759 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:13.759 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:13.760 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:13.760 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:13.760 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:50:13.760 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:50:13.760 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:50:13.760 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:50:13.760 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:50:13.761 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:50:13.761 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:50:13.761 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:50:13.761 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:50:13.761 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:50:13.761 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:50:13.761 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:50:13.761 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:50:13.761 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:50:13.761 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:50:13.761 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:50:13.761 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:50:13.761 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:13.761 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:50:13.761 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:13.762 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:13.762 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:13.762 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:13.762 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:13.762 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:50:13.762 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:13.762 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:50:13.762 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:13.762 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:50:13.762 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:13.762 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:13.762 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:13.762 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:13.762 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:13.762 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:13.763 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:13.763 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:13.763 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:14.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:50:13] "GET /metrics HTTP/1.1" 200 38144 "" "Prometheus/2.43.0" 2024-02-07T12:50:14.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-mon[88137]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 67/627 objects degraded (10.686%), 21 pgs degraded) 2024-02-07T12:50:14.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:50:14.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:14.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:14.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:14.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-02-07T12:50:14.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-mon[82918]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 67/627 objects degraded (10.686%), 21 pgs degraded) 2024-02-07T12:50:14.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:50:14.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:14.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:14.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:14.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:13 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-02-07T12:50:14.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:13 smithi172 ceph-mon[67636]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 67/627 objects degraded (10.686%), 21 pgs degraded) 2024-02-07T12:50:14.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:13 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:50:14.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:13 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:14.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:13 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:14.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:13 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:14.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:13 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-02-07T12:50:15.074 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:14 smithi019 ceph-mon[82918]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:15.074 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:50:15.074 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:15.075 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:15.075 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:15.075 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:15.075 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:15.075 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:50:15.075 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:15.075 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:50:15.075 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:15.075 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:50:15.075 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:15.075 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:15.075 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:15.075 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:15.075 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:15.076 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:15.076 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:15.076 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:15.076 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:15.076 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:50:15.076 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:50:15.076 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:50:15.076 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:50:15.076 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:50:15.076 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:50:15.076 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:50:15.076 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:50:15.076 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:50:15.076 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:50:15.076 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:50:15.077 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:50:15.077 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:50:15.077 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:50:15.077 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:50:15.077 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:50:15.077 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:50:15.077 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:15.077 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:50:15.077 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:15.077 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:15.077 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:15.077 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:15.077 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:15.077 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:50:15.077 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:15.077 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:50:15.078 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:15.078 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:50:15.078 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:15.078 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:15.078 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:15.078 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:15.078 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:15.078 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:15.078 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:15.078 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:15.078 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:15.079 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:14 smithi019 ceph-mon[82918]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-02-07T12:50:15.079 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:14 smithi019 ceph-mon[82918]: Upgrade: osd.1 is safe to restart 2024-02-07T12:50:15.080 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:14 smithi019 ceph-mon[82918]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 827 B/s rd, 0 op/s 2024-02-07T12:50:15.080 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:14 smithi019 ceph-mon[88137]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:15.080 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:50:15.080 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:15.080 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:15.080 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:15.081 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:15.081 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:15.081 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:50:15.081 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:15.081 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:50:15.081 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:15.081 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:50:15.081 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:15.081 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:15.081 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:15.081 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:15.081 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:15.081 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:15.081 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:15.081 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:15.081 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:15.082 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:50:15.082 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:50:15.082 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:50:15.082 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:50:15.082 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:50:15.082 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:50:15.082 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:50:15.082 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:50:15.082 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:50:15.082 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:50:15.082 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:50:15.082 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:50:15.082 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:50:15.082 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:50:15.082 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:50:15.083 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:50:15.083 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:50:15.083 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:15.083 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:50:15.083 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:15.083 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:15.083 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:15.083 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:15.083 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:15.083 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:50:15.083 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:15.083 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:50:15.083 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:15.083 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:50:15.083 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:15.084 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:15.084 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:15.084 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:15.084 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:15.084 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:15.084 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:15.084 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:15.084 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:15.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:14 smithi019 ceph-mon[88137]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-02-07T12:50:15.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:14 smithi019 ceph-mon[88137]: Upgrade: osd.1 is safe to restart 2024-02-07T12:50:15.084 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:14 smithi019 ceph-mon[88137]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 827 B/s rd, 0 op/s 2024-02-07T12:50:15.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:14 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:15.119 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:50:15.119 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:15.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:15.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:15.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:15.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:15.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:50:15.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:15.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:50:15.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:15.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:50:15.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:15.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:15.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:15.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:15.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:15.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:15.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:15.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:15.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:15.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:50:15.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:50:15.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:50:15.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:50:15.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:50:15.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:50:15.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:50:15.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:50:15.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:50:15.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:50:15.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:50:15.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:50:15.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:50:15.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:50:15.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:50:15.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:50:15.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:50:15.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:15.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:50:15.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:15.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:15.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:15.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:15.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:15.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:50:15.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:15.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:50:15.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:15.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:50:15.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:15.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:15.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:15.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:15.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:15.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:15.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:15.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:15.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:15.123 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:14 smithi172 ceph-mon[67636]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-02-07T12:50:15.124 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:14 smithi172 ceph-mon[67636]: Upgrade: osd.1 is safe to restart 2024-02-07T12:50:15.124 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:14 smithi172 ceph-mon[67636]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 827 B/s rd, 0 op/s 2024-02-07T12:50:16.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:16 smithi019 ceph-mon[82918]: Upgrade: Updating osd.1 2024-02-07T12:50:16.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:16 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:16.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:16 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-02-07T12:50:16.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:16 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:50:16.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:16 smithi019 ceph-mon[82918]: Deploying daemon osd.1 on smithi019 2024-02-07T12:50:16.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:16 smithi019 ceph-mon[82918]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 650 B/s rd, 0 op/s 2024-02-07T12:50:16.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:16 smithi019 ceph-mon[88137]: Upgrade: Updating osd.1 2024-02-07T12:50:16.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:16 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:16.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:16 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-02-07T12:50:16.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:16 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:50:16.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:16 smithi019 ceph-mon[88137]: Deploying daemon osd.1 on smithi019 2024-02-07T12:50:16.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:16 smithi019 ceph-mon[88137]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 650 B/s rd, 0 op/s 2024-02-07T12:50:16.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:16 smithi172 ceph-mon[67636]: Upgrade: Updating osd.1 2024-02-07T12:50:16.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:16 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:16.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:16 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-02-07T12:50:16.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:16 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:50:16.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:16 smithi172 ceph-mon[67636]: Deploying daemon osd.1 on smithi019 2024-02-07T12:50:16.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:16 smithi172 ceph-mon[67636]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 650 B/s rd, 0 op/s 2024-02-07T12:50:17.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:17 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:50:17.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:17 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:50:17.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:17 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:50:18.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:18 smithi019 ceph-mon[88137]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:50:18.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:18 smithi019 ceph-mon[82918]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:50:18.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:18 smithi172 ceph-mon[67636]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:50:18.844 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:18 smithi019 systemd[1]: Stopping Ceph osd.1 for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:50:18.844 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1[35085]: 2024-02-07T12:50:18.691+0000 7f71aaed0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:50:18.844 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1[35085]: 2024-02-07T12:50:18.691+0000 7f71aaed0700 -1 osd.1 110 *** Got signal Terminated *** 2024-02-07T12:50:18.844 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1[35085]: 2024-02-07T12:50:18.691+0000 7f71aaed0700 -1 osd.1 110 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:50:19.481 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:19 smithi019 ceph-mon[88137]: osd.1 marked itself down and dead 2024-02-07T12:50:19.481 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:19 smithi019 ceph-mon[82918]: osd.1 marked itself down and dead 2024-02-07T12:50:19.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:19 smithi172 ceph-mon[67636]: osd.1 marked itself down and dead 2024-02-07T12:50:19.844 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:19 smithi019 podman[100549]: 2024-02-07 12:50:19.479322311 +0000 UTC m=+0.880576347 container died 0f9a190f0e0dd118791febccc5f0472d5a138e6c4123022c497bb88129f2a078 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, build-date=2022-05-03T08:36:31.336870, version=8, GIT_CLEAN=True, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, ceph=True, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, architecture=x86_64, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0) 2024-02-07T12:50:20.131 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:19 smithi019 podman[100549]: 2024-02-07 12:50:19.945412601 +0000 UTC m=+1.346666644 container cleanup 0f9a190f0e0dd118791febccc5f0472d5a138e6c4123022c497bb88129f2a078 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1, RELEASE=HEAD, distribution-scope=public, architecture=x86_64, io.openshift.expose-services=, vcs-type=git, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., release=754, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True) 2024-02-07T12:50:20.131 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:19 smithi019 bash[100549]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1 2024-02-07T12:50:20.522 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:20 smithi019 podman[100567]: 2024-02-07 12:50:20.221734511 +0000 UTC m=+0.740090249 container remove 0f9a190f0e0dd118791febccc5f0472d5a138e6c4123022c497bb88129f2a078 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1, build-date=2022-05-03T08:36:31.336870, vcs-type=git, GIT_BRANCH=HEAD, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, architecture=x86_64, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, RELEASE=HEAD, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-02-07T12:50:20.523 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[88137]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:50:20.523 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[88137]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.523 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[88137]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.523 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[88137]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.523 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[88137]: osdmap e111: 8 total, 7 up, 8 in 2024-02-07T12:50:20.523 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[88137]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.523 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[88137]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.523 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[88137]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.523 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[88137]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.523 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[88137]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.523 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[88137]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.524 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[88137]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.524 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[88137]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.524 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[88137]: pgmap v85: 161 pgs: 19 stale+active+clean, 142 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-02-07T12:50:20.524 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:20.524 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:50:20.524 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:20.524 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[82918]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:50:20.524 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[82918]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.524 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[82918]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.524 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[82918]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.524 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[82918]: osdmap e111: 8 total, 7 up, 8 in 2024-02-07T12:50:20.525 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[82918]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.525 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[82918]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.525 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[82918]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.525 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[82918]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.525 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[82918]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.525 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[82918]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.525 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[82918]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.525 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[82918]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.525 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[82918]: pgmap v85: 161 pgs: 19 stale+active+clean, 142 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-02-07T12:50:20.525 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:20.525 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:50:20.525 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:20 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:20.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:20 smithi172 ceph-mon[67636]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:50:20.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:20 smithi172 ceph-mon[67636]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:20 smithi172 ceph-mon[67636]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:20 smithi172 ceph-mon[67636]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:20 smithi172 ceph-mon[67636]: osdmap e111: 8 total, 7 up, 8 in 2024-02-07T12:50:20.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:20 smithi172 ceph-mon[67636]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:20 smithi172 ceph-mon[67636]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:20 smithi172 ceph-mon[67636]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:20 smithi172 ceph-mon[67636]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:20 smithi172 ceph-mon[67636]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:20 smithi172 ceph-mon[67636]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:20 smithi172 ceph-mon[67636]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:20 smithi172 ceph-mon[67636]: failed to encode map e111 with expected crc 2024-02-07T12:50:20.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:20 smithi172 ceph-mon[67636]: pgmap v85: 161 pgs: 19 stale+active+clean, 142 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-02-07T12:50:20.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:20 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:20.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:20 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:50:20.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:20 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:20.844 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:20 smithi019 podman[100656]: 2024-02-07 12:50:20.42410009 +0000 UTC m=+0.018030995 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:50:20.845 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:20 smithi019 podman[100656]: 2024-02-07 12:50:20.522392157 +0000 UTC m=+0.116323052 container create 1e8a47ded52455d6d03924a85fb04d24020c643d41a120f34b00cb6ae6f40c6d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131) 2024-02-07T12:50:20.845 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:20 smithi019 podman[100656]: 2024-02-07 12:50:20.751219711 +0000 UTC m=+0.345150607 container init 1e8a47ded52455d6d03924a85fb04d24020c643d41a120f34b00cb6ae6f40c6d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1-deactivate, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:50:20.845 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:20 smithi019 podman[100656]: 2024-02-07 12:50:20.755185707 +0000 UTC m=+0.349116611 container start 1e8a47ded52455d6d03924a85fb04d24020c643d41a120f34b00cb6ae6f40c6d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.2, org.label-schema.vendor=CentOS) 2024-02-07T12:50:21.143 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:20 smithi019 podman[100656]: 2024-02-07 12:50:20.858396646 +0000 UTC m=+0.452327555 container attach 1e8a47ded52455d6d03924a85fb04d24020c643d41a120f34b00cb6ae6f40c6d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1-deactivate, org.label-schema.build-date=20240131, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:50:21.144 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:21 smithi019 podman[100656]: 2024-02-07 12:50:21.058759498 +0000 UTC m=+0.652690392 container died 1e8a47ded52455d6d03924a85fb04d24020c643d41a120f34b00cb6ae6f40c6d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:50:21.222 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:50:21.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[82918]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[82918]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[82918]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[82918]: osdmap e112: 8 total, 7 up, 8 in 2024-02-07T12:50:21.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[82918]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[82918]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[82918]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[82918]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[82918]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[82918]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[82918]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[82918]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[82918]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[82918]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[88137]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[88137]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[88137]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[88137]: osdmap e112: 8 total, 7 up, 8 in 2024-02-07T12:50:21.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[88137]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[88137]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[88137]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[88137]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[88137]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[88137]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[88137]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[88137]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[88137]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:21 smithi019 ceph-mon[88137]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:21 smithi172 ceph-mon[67636]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:21 smithi172 ceph-mon[67636]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:21 smithi172 ceph-mon[67636]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:21 smithi172 ceph-mon[67636]: osdmap e112: 8 total, 7 up, 8 in 2024-02-07T12:50:21.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:21 smithi172 ceph-mon[67636]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:21 smithi172 ceph-mon[67636]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:21 smithi172 ceph-mon[67636]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:21 smithi172 ceph-mon[67636]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:21 smithi172 ceph-mon[67636]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:21 smithi172 ceph-mon[67636]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:21 smithi172 ceph-mon[67636]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:21 smithi172 ceph-mon[67636]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:21 smithi172 ceph-mon[67636]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:21 smithi172 ceph-mon[67636]: failed to encode map e112 with expected crc 2024-02-07T12:50:21.882 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:21 smithi019 podman[100705]: 2024-02-07 12:50:21.62934194 +0000 UTC m=+0.560403911 container remove 1e8a47ded52455d6d03924a85fb04d24020c643d41a120f34b00cb6ae6f40c6d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True) 2024-02-07T12:50:21.882 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:21 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.1.service: Deactivated successfully. 2024-02-07T12:50:21.882 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:21 smithi019 systemd[1]: Stopped Ceph osd.1 for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:50:21.882 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:21 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.1.service: Consumed 14.896s CPU time. 2024-02-07T12:50:21.882 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:21 smithi019 systemd[1]: Starting Ceph osd.1 for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:50:21.940 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:50:21.940 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (4m) 14s ago 26m 27.9M - 0.25.0 c8568f914cd2 fec9255f35e7 2024-02-07T12:50:21.940 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (4m) 2m ago 26m 82.9M - 9.4.12 a0ea5e3c46d9 aaffac63bc28 2024-02-07T12:50:21.940 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 unknown 14s ago 25m 63.3M - 3.5 e1d6a67b021e 5113aa77e3f2 2024-02-07T12:50:21.941 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283,8765 running (5m) 2m ago 30m 447M - 19.0.0-1252-g44458cce 79957b372307 5aa0c04fdfd7 2024-02-07T12:50:21.941 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (15m) 14s ago 32m 502M - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:50:21.941 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (2m) 14s ago 32m 46.5M 2048M 19.0.0-1252-g44458cce 79957b372307 8b1b0e21b1c5 2024-02-07T12:50:21.941 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (3m) 2m ago 31m 37.0M 2048M 19.0.0-1252-g44458cce 79957b372307 00f454bbac01 2024-02-07T12:50:21.941 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (119s) 14s ago 31m 38.0M 2048M 19.0.0-1252-g44458cce 79957b372307 6214003f6e08 2024-02-07T12:50:21.941 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (5m) 14s ago 27m 16.0M - 1.5.0 0da6a335fe13 c4bdcb1c4d49 2024-02-07T12:50:21.941 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (5m) 2m ago 27m 16.3M - 1.5.0 0da6a335fe13 04ea81f79d85 2024-02-07T12:50:21.941 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (19s) 14s ago 30m 15.0M 2993M 19.0.0-1252-g44458cce 79957b372307 78eddd34a903 2024-02-07T12:50:21.941 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (30m) 14s ago 30m 58.3M 2993M 17.2.0 e1d6a67b021e 0f9a190f0e0d 2024-02-07T12:50:21.941 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (29m) 14s ago 29m 54.4M 2993M 17.2.0 e1d6a67b021e 7db94f25a1aa 2024-02-07T12:50:21.941 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (29m) 14s ago 29m 54.5M 2993M 17.2.0 e1d6a67b021e 0b6b674188ed 2024-02-07T12:50:21.941 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (29m) 2m ago 29m 55.7M 3248M 17.2.0 e1d6a67b021e df72cfd69b3b 2024-02-07T12:50:21.942 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (28m) 2m ago 28m 52.4M 3248M 17.2.0 e1d6a67b021e 3bdb22c07537 2024-02-07T12:50:21.942 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (28m) 2m ago 28m 52.1M 3248M 17.2.0 e1d6a67b021e 7859b651e0d9 2024-02-07T12:50:21.942 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (27m) 2m ago 27m 55.5M 3248M 17.2.0 e1d6a67b021e b301e7c75d73 2024-02-07T12:50:21.942 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (4m) 2m ago 27m 61.0M - 2.43.0 a07b618ecd1d c9b607eea1be 2024-02-07T12:50:21.942 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (25m) 14s ago 25m 79.2M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:50:21.942 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (25m) 2m ago 25m 78.3M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:50:22.149 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:22 smithi019 podman[100859]: 2024-02-07 12:50:22.088525127 +0000 UTC m=+0.105615719 container create d49c88e09b3fa058ee90ecd1475b7c81a97a8495e0cd21c599b5d8f7f63a164c (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1-activate, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131, org.label-schema.license=GPLv2) 2024-02-07T12:50:22.149 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:22 smithi019 podman[100859]: 2024-02-07 12:50:22.001418182 +0000 UTC m=+0.018508785 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:50:22.352 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:50:22.352 INFO:teuthology.orchestra.run.smithi019.stdout: "mon": { 2024-02-07T12:50:22.352 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 3 2024-02-07T12:50:22.352 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:50:22.352 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr": { 2024-02-07T12:50:22.352 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:50:22.352 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:50:22.352 INFO:teuthology.orchestra.run.smithi019.stdout: "osd": { 2024-02-07T12:50:22.352 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-02-07T12:50:22.352 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 1 2024-02-07T12:50:22.352 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:50:22.353 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw": { 2024-02-07T12:50:22.353 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:50:22.353 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:50:22.353 INFO:teuthology.orchestra.run.smithi019.stdout: "overall": { 2024-02-07T12:50:22.353 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-02-07T12:50:22.353 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 6 2024-02-07T12:50:22.353 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:50:22.353 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:50:22.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:22 smithi019 ceph-mon[88137]: from='client.44280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:50:22.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:22 smithi019 ceph-mon[88137]: from='client.54223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:50:22.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:22 smithi019 ceph-mon[88137]: pgmap v87: 161 pgs: 22 active+undersized+degraded, 41 active+undersized, 98 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 85/627 objects degraded (13.557%) 2024-02-07T12:50:22.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:22 smithi019 ceph-mon[88137]: from='client.44292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:50:22.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:22 smithi019 ceph-mon[82918]: from='client.44280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:50:22.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:22 smithi019 ceph-mon[82918]: from='client.54223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:50:22.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:22 smithi019 ceph-mon[82918]: pgmap v87: 161 pgs: 22 active+undersized+degraded, 41 active+undersized, 98 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 85/627 objects degraded (13.557%) 2024-02-07T12:50:22.596 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:22 smithi019 ceph-mon[82918]: from='client.44292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:50:22.596 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:22 smithi019 podman[100859]: 2024-02-07 12:50:22.299762861 +0000 UTC m=+0.316853482 container init d49c88e09b3fa058ee90ecd1475b7c81a97a8495e0cd21c599b5d8f7f63a164c (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1-activate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-02-07T12:50:22.596 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:22 smithi019 podman[100859]: 2024-02-07 12:50:22.303078983 +0000 UTC m=+0.320169571 container start d49c88e09b3fa058ee90ecd1475b7c81a97a8495e0cd21c599b5d8f7f63a164c (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux ) 2024-02-07T12:50:22.596 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:22 smithi019 podman[100859]: 2024-02-07 12:50:22.365971237 +0000 UTC m=+0.383061823 container attach d49c88e09b3fa058ee90ecd1475b7c81a97a8495e0cd21c599b5d8f7f63a164c (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1-activate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:50:22.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:22 smithi172 ceph-mon[67636]: from='client.44280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:50:22.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:22 smithi172 ceph-mon[67636]: from='client.54223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:50:22.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:22 smithi172 ceph-mon[67636]: pgmap v87: 161 pgs: 22 active+undersized+degraded, 41 active+undersized, 98 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 85/627 objects degraded (13.557%) 2024-02-07T12:50:22.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:22 smithi172 ceph-mon[67636]: from='client.44292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:50:22.710 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:50:22.710 INFO:teuthology.orchestra.run.smithi019.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:50:22.710 INFO:teuthology.orchestra.run.smithi019.stdout: "in_progress": true, 2024-02-07T12:50:22.710 INFO:teuthology.orchestra.run.smithi019.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (1 remaining).", 2024-02-07T12:50:22.711 INFO:teuthology.orchestra.run.smithi019.stdout: "services_complete": [], 2024-02-07T12:50:22.711 INFO:teuthology.orchestra.run.smithi019.stdout: "progress": "1/8 daemons upgraded", 2024-02-07T12:50:22.711 INFO:teuthology.orchestra.run.smithi019.stdout: "message": "Currently upgrading osd daemons", 2024-02-07T12:50:22.711 INFO:teuthology.orchestra.run.smithi019.stdout: "is_paused": false 2024-02-07T12:50:22.711 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:50:23.080 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:22 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1-activate[100894]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-02-07T12:50:23.080 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:22 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1-activate[100894]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-02-07T12:50:23.080 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:22 smithi019 bash[100859]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-02-07T12:50:23.080 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:22 smithi019 bash[100859]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-02-07T12:50:23.080 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:22 smithi019 bash[100859]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-02-07T12:50:23.080 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:22 smithi019 bash[100859]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-02-07T12:50:23.081 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:22 smithi019 bash[100859]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-02-07T12:50:23.081 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:22 smithi019 bash[100859]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-02-07T12:50:23.081 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:22 smithi019 bash[100859]: --> ceph-volume raw activate successful for osd ID: 1 2024-02-07T12:50:23.081 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:22 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1-activate[100894]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-02-07T12:50:23.081 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:22 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1-activate[100894]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-02-07T12:50:23.081 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:22 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1-activate[100894]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-02-07T12:50:23.081 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:22 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1-activate[100894]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-02-07T12:50:23.081 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:22 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1-activate[100894]: --> ceph-volume raw activate successful for osd ID: 1 2024-02-07T12:50:23.081 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:22 smithi019 podman[100859]: 2024-02-07 12:50:22.788375686 +0000 UTC m=+0.805466273 container died d49c88e09b3fa058ee90ecd1475b7c81a97a8495e0cd21c599b5d8f7f63a164c (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1-activate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.2, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240131) 2024-02-07T12:50:23.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:23 smithi019 ceph-mon[88137]: Health check failed: Degraded data redundancy: 85/627 objects degraded (13.557%), 22 pgs degraded (PG_DEGRADED) 2024-02-07T12:50:23.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:23 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/2034246655' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:23.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:23 smithi019 ceph-mon[88137]: from='client.54241 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:50:23.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:23 smithi019 ceph-mon[82918]: Health check failed: Degraded data redundancy: 85/627 objects degraded (13.557%), 22 pgs degraded (PG_DEGRADED) 2024-02-07T12:50:23.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:23 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/2034246655' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:23.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:23 smithi019 ceph-mon[82918]: from='client.54241 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:50:23.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:23 smithi172 ceph-mon[67636]: Health check failed: Degraded data redundancy: 85/627 objects degraded (13.557%), 22 pgs degraded (PG_DEGRADED) 2024-02-07T12:50:23.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:23 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/2034246655' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:23.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:23 smithi172 ceph-mon[67636]: from='client.54241 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:50:23.736 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:23 smithi019 podman[101039]: 2024-02-07 12:50:23.38676277 +0000 UTC m=+0.592822971 container remove d49c88e09b3fa058ee90ecd1475b7c81a97a8495e0cd21c599b5d8f7f63a164c (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1-activate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240131, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.2) 2024-02-07T12:50:23.736 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:23 smithi019 podman[101106]: 2024-02-07 12:50:23.504972759 +0000 UTC m=+0.017620596 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:50:23.736 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:23 smithi019 podman[101106]: 2024-02-07 12:50:23.620813136 +0000 UTC m=+0.133460964 container create c37447e76c38ce3e6d1d82848733c1563d4ce9dbfcd2868b84d5992d49912101 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1, ceph=True, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-02-07T12:50:24.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:50:23] "GET /metrics HTTP/1.1" 200 38167 "" "Prometheus/2.43.0" 2024-02-07T12:50:24.094 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:23 smithi019 podman[101106]: 2024-02-07 12:50:23.865867853 +0000 UTC m=+0.378515687 container init c37447e76c38ce3e6d1d82848733c1563d4ce9dbfcd2868b84d5992d49912101 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240131, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-02-07T12:50:24.094 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:23 smithi019 podman[101106]: 2024-02-07 12:50:23.869210373 +0000 UTC m=+0.381858201 container start c37447e76c38ce3e6d1d82848733c1563d4ce9dbfcd2868b84d5992d49912101 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-02-07T12:50:24.094 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:23 smithi019 ceph-osd[101143]: -- 172.21.15.19:0/228253464 <== mon.0 v2:172.21.15.19:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5599f86936c0 con 0x5599f8665400 2024-02-07T12:50:24.094 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:23 smithi019 bash[101106]: c37447e76c38ce3e6d1d82848733c1563d4ce9dbfcd2868b84d5992d49912101 2024-02-07T12:50:24.095 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:23 smithi019 systemd[1]: Started Ceph osd.1 for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:50:24.414 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:24 smithi019 ceph-mon[82918]: pgmap v88: 161 pgs: 22 active+undersized+degraded, 41 active+undersized, 98 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 85/627 objects degraded (13.557%) 2024-02-07T12:50:24.414 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:24 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:24.414 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:24 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:24.414 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:24 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:24.414 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:24 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:50:24.414 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:24 smithi019 ceph-mon[88137]: pgmap v88: 161 pgs: 22 active+undersized+degraded, 41 active+undersized, 98 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 85/627 objects degraded (13.557%) 2024-02-07T12:50:24.415 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:24 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:24.415 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:24 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:24.415 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:24 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:24.415 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:24 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:50:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:24 smithi172 ceph-mon[67636]: pgmap v88: 161 pgs: 22 active+undersized+degraded, 41 active+undersized, 98 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 85/627 objects degraded (13.557%) 2024-02-07T12:50:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:24 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:24 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:24 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:24 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:50:24.830 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:24 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1[101137]: 2024-02-07T12:50:24.411+0000 7f1b1f87f7c0 -1 Falling back to public interface 2024-02-07T12:50:27.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:26 smithi172 ceph-mon[67636]: pgmap v89: 161 pgs: 22 active+undersized+degraded, 41 active+undersized, 98 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 85/627 objects degraded (13.557%) 2024-02-07T12:50:27.135 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:26 smithi019 ceph-mon[88137]: pgmap v89: 161 pgs: 22 active+undersized+degraded, 41 active+undersized, 98 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 85/627 objects degraded (13.557%) 2024-02-07T12:50:27.135 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:26 smithi019 ceph-mon[82918]: pgmap v89: 161 pgs: 22 active+undersized+degraded, 41 active+undersized, 98 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 85/627 objects degraded (13.557%) 2024-02-07T12:50:27.594 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:27 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1[101137]: 2024-02-07T12:50:27.132+0000 7f1b1f87f7c0 -1 osd.1 0 read_superblock omap replica is missing. 2024-02-07T12:50:27.594 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:27 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1[101137]: 2024-02-07T12:50:27.172+0000 7f1b1f87f7c0 -1 osd.1 110 log_to_monitors true 2024-02-07T12:50:27.982 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:27 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:50:27.983 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:27 smithi019 ceph-mon[88137]: from='osd.1 [v2:172.21.15.19:6810/1959086602,v1:172.21.15.19:6811/1959086602]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-02-07T12:50:27.983 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:27 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:50:27.983 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:27 smithi019 ceph-mon[82918]: from='osd.1 [v2:172.21.15.19:6810/1959086602,v1:172.21.15.19:6811/1959086602]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-02-07T12:50:27.983 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:50:27 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1[101137]: 2024-02-07T12:50:27.897+0000 7f1b10cde700 -1 osd.1 110 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:50:28.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:27 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:50:28.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:27 smithi172 ceph-mon[67636]: from='osd.1 [v2:172.21.15.19:6810/1959086602,v1:172.21.15.19:6811/1959086602]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-02-07T12:50:29.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:28 smithi019 ceph-mon[88137]: pgmap v90: 161 pgs: 22 active+undersized+degraded, 41 active+undersized, 98 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 85/627 objects degraded (13.557%) 2024-02-07T12:50:29.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:28 smithi019 ceph-mon[88137]: from='osd.1 [v2:172.21.15.19:6810/1959086602,v1:172.21.15.19:6811/1959086602]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-02-07T12:50:29.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:28 smithi019 ceph-mon[88137]: failed to encode map e113 with expected crc 2024-02-07T12:50:29.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:28 smithi019 ceph-mon[88137]: failed to encode map e113 with expected crc 2024-02-07T12:50:29.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:28 smithi019 ceph-mon[88137]: osdmap e113: 8 total, 7 up, 8 in 2024-02-07T12:50:29.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:28 smithi019 ceph-mon[88137]: from='osd.1 [v2:172.21.15.19:6810/1959086602,v1:172.21.15.19:6811/1959086602]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:50:29.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:28 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:29.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:28 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:29.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:28 smithi019 ceph-mon[88137]: failed to encode map e113 with expected crc 2024-02-07T12:50:29.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:28 smithi019 ceph-mon[88137]: failed to encode map e113 with expected crc 2024-02-07T12:50:29.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:28 smithi019 ceph-mon[82918]: pgmap v90: 161 pgs: 22 active+undersized+degraded, 41 active+undersized, 98 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 85/627 objects degraded (13.557%) 2024-02-07T12:50:29.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:28 smithi019 ceph-mon[82918]: from='osd.1 [v2:172.21.15.19:6810/1959086602,v1:172.21.15.19:6811/1959086602]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-02-07T12:50:29.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:28 smithi019 ceph-mon[82918]: failed to encode map e113 with expected crc 2024-02-07T12:50:29.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:28 smithi019 ceph-mon[82918]: failed to encode map e113 with expected crc 2024-02-07T12:50:29.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:28 smithi019 ceph-mon[82918]: osdmap e113: 8 total, 7 up, 8 in 2024-02-07T12:50:29.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:28 smithi019 ceph-mon[82918]: from='osd.1 [v2:172.21.15.19:6810/1959086602,v1:172.21.15.19:6811/1959086602]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:50:29.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:28 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:29.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:28 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:29.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:28 smithi019 ceph-mon[82918]: failed to encode map e113 with expected crc 2024-02-07T12:50:29.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:28 smithi019 ceph-mon[82918]: failed to encode map e113 with expected crc 2024-02-07T12:50:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:28 smithi172 ceph-mon[67636]: pgmap v90: 161 pgs: 22 active+undersized+degraded, 41 active+undersized, 98 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 85/627 objects degraded (13.557%) 2024-02-07T12:50:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:28 smithi172 ceph-mon[67636]: from='osd.1 [v2:172.21.15.19:6810/1959086602,v1:172.21.15.19:6811/1959086602]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-02-07T12:50:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:28 smithi172 ceph-mon[67636]: failed to encode map e113 with expected crc 2024-02-07T12:50:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:28 smithi172 ceph-mon[67636]: failed to encode map e113 with expected crc 2024-02-07T12:50:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:28 smithi172 ceph-mon[67636]: osdmap e113: 8 total, 7 up, 8 in 2024-02-07T12:50:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:28 smithi172 ceph-mon[67636]: from='osd.1 [v2:172.21.15.19:6810/1959086602,v1:172.21.15.19:6811/1959086602]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:50:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:28 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:28 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:28 smithi172 ceph-mon[67636]: failed to encode map e113 with expected crc 2024-02-07T12:50:29.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:28 smithi172 ceph-mon[67636]: failed to encode map e113 with expected crc 2024-02-07T12:50:30.204 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[82918]: failed to encode map e113 with expected crc 2024-02-07T12:50:30.204 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[82918]: failed to encode map e113 with expected crc 2024-02-07T12:50:30.204 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[82918]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:50:30.204 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[82918]: failed to encode map e114 with expected crc 2024-02-07T12:50:30.204 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[82918]: failed to encode map e114 with expected crc 2024-02-07T12:50:30.204 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:50:30.204 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[82918]: osd.1 [v2:172.21.15.19:6810/1959086602,v1:172.21.15.19:6811/1959086602] boot 2024-02-07T12:50:30.204 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[82918]: osdmap e114: 8 total, 8 up, 8 in 2024-02-07T12:50:30.204 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[82918]: failed to encode map e114 with expected crc 2024-02-07T12:50:30.205 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[82918]: failed to encode map e114 with expected crc 2024-02-07T12:50:30.205 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[82918]: failed to encode map e114 with expected crc 2024-02-07T12:50:30.205 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[82918]: failed to encode map e114 with expected crc 2024-02-07T12:50:30.205 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[82918]: failed to encode map e114 with expected crc 2024-02-07T12:50:30.205 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:30.205 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:30.205 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[82918]: osdmap e115: 8 total, 8 up, 8 in 2024-02-07T12:50:30.205 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[88137]: failed to encode map e113 with expected crc 2024-02-07T12:50:30.205 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[88137]: failed to encode map e113 with expected crc 2024-02-07T12:50:30.205 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[88137]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:50:30.205 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[88137]: failed to encode map e114 with expected crc 2024-02-07T12:50:30.205 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[88137]: failed to encode map e114 with expected crc 2024-02-07T12:50:30.205 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:50:30.206 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[88137]: osd.1 [v2:172.21.15.19:6810/1959086602,v1:172.21.15.19:6811/1959086602] boot 2024-02-07T12:50:30.206 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[88137]: osdmap e114: 8 total, 8 up, 8 in 2024-02-07T12:50:30.206 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[88137]: failed to encode map e114 with expected crc 2024-02-07T12:50:30.206 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[88137]: failed to encode map e114 with expected crc 2024-02-07T12:50:30.206 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[88137]: failed to encode map e114 with expected crc 2024-02-07T12:50:30.206 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[88137]: failed to encode map e114 with expected crc 2024-02-07T12:50:30.206 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[88137]: failed to encode map e114 with expected crc 2024-02-07T12:50:30.206 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:30.206 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:30.206 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:29 smithi019 ceph-mon[88137]: osdmap e115: 8 total, 8 up, 8 in 2024-02-07T12:50:30.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:29 smithi172 ceph-mon[67636]: failed to encode map e113 with expected crc 2024-02-07T12:50:30.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:29 smithi172 ceph-mon[67636]: failed to encode map e113 with expected crc 2024-02-07T12:50:30.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:29 smithi172 ceph-mon[67636]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:50:30.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:29 smithi172 ceph-mon[67636]: failed to encode map e114 with expected crc 2024-02-07T12:50:30.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:29 smithi172 ceph-mon[67636]: failed to encode map e114 with expected crc 2024-02-07T12:50:30.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:29 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:50:30.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:29 smithi172 ceph-mon[67636]: osd.1 [v2:172.21.15.19:6810/1959086602,v1:172.21.15.19:6811/1959086602] boot 2024-02-07T12:50:30.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:29 smithi172 ceph-mon[67636]: osdmap e114: 8 total, 8 up, 8 in 2024-02-07T12:50:30.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:29 smithi172 ceph-mon[67636]: failed to encode map e114 with expected crc 2024-02-07T12:50:30.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:29 smithi172 ceph-mon[67636]: failed to encode map e114 with expected crc 2024-02-07T12:50:30.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:29 smithi172 ceph-mon[67636]: failed to encode map e114 with expected crc 2024-02-07T12:50:30.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:29 smithi172 ceph-mon[67636]: failed to encode map e114 with expected crc 2024-02-07T12:50:30.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:29 smithi172 ceph-mon[67636]: failed to encode map e114 with expected crc 2024-02-07T12:50:30.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:29 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:30.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:29 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:30.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:29 smithi172 ceph-mon[67636]: osdmap e115: 8 total, 8 up, 8 in 2024-02-07T12:50:31.007 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[88137]: pgmap v93: 161 pgs: 22 active+undersized+degraded, 41 active+undersized, 98 active+clean; 585 KiB data, 130 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 85/627 objects degraded (13.557%) 2024-02-07T12:50:31.008 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[88137]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.008 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[88137]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.008 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[88137]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.008 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[88137]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.008 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[88137]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.008 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[88137]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.008 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[88137]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.008 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[88137]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.008 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[88137]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.008 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[88137]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.008 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[88137]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.008 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[88137]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.008 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[88137]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.009 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[88137]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.009 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[88137]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.009 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[82918]: pgmap v93: 161 pgs: 22 active+undersized+degraded, 41 active+undersized, 98 active+clean; 585 KiB data, 130 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 85/627 objects degraded (13.557%) 2024-02-07T12:50:31.009 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[82918]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.009 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[82918]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.009 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[82918]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.009 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[82918]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.009 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[82918]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.009 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[82918]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.009 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[82918]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.009 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[82918]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.009 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[82918]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.010 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[82918]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.010 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[82918]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.010 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[82918]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.010 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[82918]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.010 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[82918]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.010 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:30 smithi019 ceph-mon[82918]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:30 smithi172 ceph-mon[67636]: pgmap v93: 161 pgs: 22 active+undersized+degraded, 41 active+undersized, 98 active+clean; 585 KiB data, 130 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 85/627 objects degraded (13.557%) 2024-02-07T12:50:31.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:30 smithi172 ceph-mon[67636]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:30 smithi172 ceph-mon[67636]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:30 smithi172 ceph-mon[67636]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:30 smithi172 ceph-mon[67636]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:30 smithi172 ceph-mon[67636]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:30 smithi172 ceph-mon[67636]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:30 smithi172 ceph-mon[67636]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:30 smithi172 ceph-mon[67636]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:30 smithi172 ceph-mon[67636]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:30 smithi172 ceph-mon[67636]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:30 smithi172 ceph-mon[67636]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:30 smithi172 ceph-mon[67636]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:30 smithi172 ceph-mon[67636]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:30 smithi172 ceph-mon[67636]: failed to encode map e115 with expected crc 2024-02-07T12:50:31.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:30 smithi172 ceph-mon[67636]: failed to encode map e115 with expected crc 2024-02-07T12:50:32.199 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:31 smithi019 ceph-mon[82918]: Health check update: Degraded data redundancy: 30/627 objects degraded (4.785%), 6 pgs degraded (PG_DEGRADED) 2024-02-07T12:50:32.200 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:31 smithi019 ceph-mon[88137]: Health check update: Degraded data redundancy: 30/627 objects degraded (4.785%), 6 pgs degraded (PG_DEGRADED) 2024-02-07T12:50:32.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:31 smithi172 ceph-mon[67636]: Health check update: Degraded data redundancy: 30/627 objects degraded (4.785%), 6 pgs degraded (PG_DEGRADED) 2024-02-07T12:50:33.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:32 smithi019 ceph-mon[82918]: pgmap v95: 161 pgs: 6 active+undersized+degraded, 13 active+undersized, 142 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 30/627 objects degraded (4.785%) 2024-02-07T12:50:33.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:32 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:33.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:32 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:33.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:32 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:50:33.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:32 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:50:33.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:32 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:33.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:32 smithi019 ceph-mon[88137]: pgmap v95: 161 pgs: 6 active+undersized+degraded, 13 active+undersized, 142 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 30/627 objects degraded (4.785%) 2024-02-07T12:50:33.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:32 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:33.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:32 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:33.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:32 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:50:33.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:32 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:50:33.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:32 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:32 smithi172 ceph-mon[67636]: pgmap v95: 161 pgs: 6 active+undersized+degraded, 13 active+undersized, 142 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 30/627 objects degraded (4.785%) 2024-02-07T12:50:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:32 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:32 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:32 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:50:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:32 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:50:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:32 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:33.951 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:50:33] "GET /metrics HTTP/1.1" 200 38167 "" "Prometheus/2.43.0" 2024-02-07T12:50:34.294 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:33 smithi019 ceph-mon[88137]: pgmap v96: 161 pgs: 6 active+undersized+degraded, 13 active+undersized, 142 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1011 B/s rd, 0 op/s; 30/627 objects degraded (4.785%) 2024-02-07T12:50:34.294 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:33 smithi019 ceph-mon[88137]: pgmap v97: 161 pgs: 6 active+undersized+degraded, 13 active+undersized, 142 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 30/627 objects degraded (4.785%) 2024-02-07T12:50:34.294 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:33 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:50:34.294 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:33 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:50:34.294 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:33 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:50:34.295 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:33 smithi019 ceph-mon[82918]: pgmap v96: 161 pgs: 6 active+undersized+degraded, 13 active+undersized, 142 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1011 B/s rd, 0 op/s; 30/627 objects degraded (4.785%) 2024-02-07T12:50:34.295 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:33 smithi019 ceph-mon[82918]: pgmap v97: 161 pgs: 6 active+undersized+degraded, 13 active+undersized, 142 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 30/627 objects degraded (4.785%) 2024-02-07T12:50:34.295 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:33 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:50:34.295 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:33 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:50:34.296 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:33 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:50:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:33 smithi172 ceph-mon[67636]: pgmap v96: 161 pgs: 6 active+undersized+degraded, 13 active+undersized, 142 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1011 B/s rd, 0 op/s; 30/627 objects degraded (4.785%) 2024-02-07T12:50:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:33 smithi172 ceph-mon[67636]: pgmap v97: 161 pgs: 6 active+undersized+degraded, 13 active+undersized, 142 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 30/627 objects degraded (4.785%) 2024-02-07T12:50:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:33 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:50:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:33 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:50:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:33 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:50:34.844 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:50:34.737+0000 7fb71d671700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:34.844 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:50:34.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:34.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:34.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:34.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:34.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:34.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:50:34.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:34.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:50:34.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:34.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:50:34.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:34.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:34.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:34.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:34.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:34.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:34.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:34.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:34.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:34.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:50:34.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:50:34.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:50:34.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:50:34.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:50:34.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:50:34.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:50:34.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:50:34.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:50:34.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:50:34.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:50:34.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:50:34.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:50:34.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:50:34.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:50:34.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:50:34.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:50:34.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:34.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:50:34.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:34.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:34.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:34.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:34.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:34.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:50:34.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:34.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:50:34.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:34.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:50:34.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:34.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:34.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:34.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:34.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:34.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:34.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:34.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:34.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:35.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:50:35.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:50:35.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:50:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:50:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:50:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-02-07T12:50:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-02-07T12:50:35.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:50:35.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:50:35.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-02-07T12:50:35.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:50:35.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:50:35.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:50:35.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:50:35.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:50:35.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:50:35.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:50:35.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:50:35.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:35.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:35.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:35.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:35.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:35.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:35.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:35.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:35.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:35.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:50:35.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:50:35.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-02-07T12:50:35.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-02-07T12:50:35.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:50:35.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:50:35.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-02-07T12:50:35.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:50:35.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:50:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:50:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:50:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:50:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:50:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:50:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:50:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:50:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:50:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:50:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:50:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:50:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:50:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:50:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:50:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-02-07T12:50:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.351 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:50:35.351 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:50:35.351 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:50:35.351 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.351 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.351 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:50:35.351 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:50:35.351 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:50:35.351 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:50:35.351 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:50:35.351 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:50:35.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:50:35.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:50:35.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-02-07T12:50:35.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:50:35.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:50:35.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-02-07T12:50:35.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:50:35.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:50:35.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:50:35.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:35.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:50:35.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:50:35.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-02-07T12:50:35.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:50:35.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:50:35.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:50:35.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:34 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:50:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:50:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:50:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:50:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:50:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-02-07T12:50:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-02-07T12:50:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:50:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:50:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-02-07T12:50:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:50:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:50:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:50:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:50:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:50:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:50:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:50:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:50:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:50:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:50:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:50:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:50:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:50:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:50:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:50:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:50:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-02-07T12:50:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:50:35.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:50:35.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:50:35.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-02-07T12:50:35.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:50:35.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:50:35.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:50:35.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:34 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:36.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[88137]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:36.344 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:50:36.344 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:36.344 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:36.344 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:36.344 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:36.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:36.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:50:36.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:36.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:50:36.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:36.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:50:36.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:36.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:36.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:36.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:36.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:36.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:36.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:36.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:36.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:36.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:50:36.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:50:36.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:50:36.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:50:36.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:50:36.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:50:36.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:50:36.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:50:36.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:50:36.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:50:36.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:50:36.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:50:36.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:50:36.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:50:36.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:50:36.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:50:36.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:50:36.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:36.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:50:36.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:36.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:36.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:36.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:36.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:36.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:50:36.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:36.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:50:36.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:36.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:50:36.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:36.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:36.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:36.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:36.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[82918]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:36.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:50:36.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:36.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:36.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:36.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:36.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:36.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:50:36.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:36.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:50:36.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:36.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:50:36.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:36.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:36.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:36.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:36.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:36.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:36.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:36.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:36.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:36.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:50:36.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:50:36.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:50:36.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:50:36.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:50:36.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:50:36.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:50:36.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:50:36.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:50:36.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:50:36.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:50:36.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:50:36.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:50:36.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:50:36.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:50:36.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:50:36.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:50:36.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:36.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:50:36.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:36.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:36.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:36.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:36.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:36.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:50:36.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:36.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:50:36.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:36.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:50:36.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:36.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:36.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:36.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:36.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:36.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:36.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:36.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:36.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:36.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all nvmeof 2024-02-07T12:50:36.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[82918]: Upgrade: Finalizing container_image settings 2024-02-07T12:50:36.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[82918]: Upgrade: Complete! 2024-02-07T12:50:36.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[82918]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 861 B/s rd, 0 op/s 2024-02-07T12:50:36.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[82918]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1014 B/s rd, 0 op/s 2024-02-07T12:50:36.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[82918]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:50:36.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:50:36.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:50:36.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:50:36.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[82918]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:50:36.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:36.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:50:36.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:36.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[82918]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/627 objects degraded (4.785%), 6 pgs degraded) 2024-02-07T12:50:36.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:36.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:36.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:36.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:36.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:36.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all nvmeof 2024-02-07T12:50:36.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[88137]: Upgrade: Finalizing container_image settings 2024-02-07T12:50:36.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[88137]: Upgrade: Complete! 2024-02-07T12:50:36.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[88137]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 861 B/s rd, 0 op/s 2024-02-07T12:50:36.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[88137]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1014 B/s rd, 0 op/s 2024-02-07T12:50:36.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[88137]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:50:36.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:50:36.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:50:36.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:50:36.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[88137]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:50:36.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:36.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:50:36.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:36.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:35 smithi019 ceph-mon[88137]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/627 objects degraded (4.785%), 6 pgs degraded) 2024-02-07T12:50:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:35 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:50:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:50:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:36.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:50:36.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:36.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:50:36.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:36.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:36.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:36.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:36.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:36.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:36.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:36.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:36.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:36.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:50:36.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:50:36.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:50:36.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:50:36.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:50:36.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:50:36.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:50:36.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:50:36.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:50:36.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:50:36.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:50:36.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:50:36.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:50:36.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:50:36.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:50:36.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:50:36.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:50:36.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:36.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:50:36.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:36.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:36.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:36.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:36.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:36.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:50:36.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:36.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:50:36.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:36.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:50:36.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:36.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:36.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:36.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:36.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:36.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:36.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:36.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:36.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:36.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:35 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all nvmeof 2024-02-07T12:50:36.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:35 smithi172 ceph-mon[67636]: Upgrade: Finalizing container_image settings 2024-02-07T12:50:36.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:35 smithi172 ceph-mon[67636]: Upgrade: Complete! 2024-02-07T12:50:36.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:35 smithi172 ceph-mon[67636]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 861 B/s rd, 0 op/s 2024-02-07T12:50:36.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:35 smithi172 ceph-mon[67636]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1014 B/s rd, 0 op/s 2024-02-07T12:50:36.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:35 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:50:36.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:35 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:50:36.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:35 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:50:36.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:35 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:50:36.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:35 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:50:36.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:35 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:36.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:35 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:50:36.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:35 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:36.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:35 smithi172 ceph-mon[67636]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/627 objects degraded (4.785%), 6 pgs degraded) 2024-02-07T12:50:37.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:50:37.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:50:37.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:50:37.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:37.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:50:37.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:50:37.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:50:37.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:37.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:50:36.876+0000 7fb71d671700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:37.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:50:37.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:37.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:37.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:37.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:37.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:37.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:50:37.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:37.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:50:37.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:37.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:50:37.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:37.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:37.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:37.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:37.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:37.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:37.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:37.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:37.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:37.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:50:37.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:50:37.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:50:37.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:50:37.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:50:37.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:50:37.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:50:37.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:50:37.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:50:37.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:50:37.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:50:37.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:50:37.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:50:37.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:50:37.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:50:37.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:50:37.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:50:37.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:37.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:50:37.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:37.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:37.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:37.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:37.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:37.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:50:37.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:37.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:50:37.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:37.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:50:37.100 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:37.100 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:37.100 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:37.100 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:37.100 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:37.100 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:37.100 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:37.100 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:37.100 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:36 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:37.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:36 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:50:37.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:36 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:50:37.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:36 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:50:37.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:36 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:38.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:37 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:50:38.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:37 smithi019 ceph-mon[88137]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-02-07T12:50:38.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:37 smithi019 ceph-mon[88137]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:38.344 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:50:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:50:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:50:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:50:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:50:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:50:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:50:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:50:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:50:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:50:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:50:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:50:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:50:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:50:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:50:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:50:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:50:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:50:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:50:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:50:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:50:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:50:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:50:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:38.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:50:38.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:38.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:50:38.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:38.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:38.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:38.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:37 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:50:38.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:37 smithi019 ceph-mon[82918]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-02-07T12:50:38.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:37 smithi019 ceph-mon[82918]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:38.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:50:38.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:50:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:50:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:50:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:38.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:38.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:38.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:50:38.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:50:38.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:50:38.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:50:38.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:50:38.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:50:38.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:50:38.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:50:38.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:50:38.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:50:38.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:50:38.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:50:38.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:50:38.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:50:38.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:50:38.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:50:38.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:50:38.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:38.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:50:38.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:38.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:38.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:38.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:38.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:38.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:50:38.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:38.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:50:38.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:38.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:50:38.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:38.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:38.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:38.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:38.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:38.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:38.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:38.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:38.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:38.352 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:37 smithi019 ceph-mon[88137]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1000 B/s rd, 0 op/s 2024-02-07T12:50:38.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:37 smithi019 ceph-mon[88137]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:50:38.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:50:38.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:50:38.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:50:38.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:38.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:38.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:38.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:38.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:38.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:38.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:37 smithi019 ceph-mon[82918]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1000 B/s rd, 0 op/s 2024-02-07T12:50:38.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:37 smithi019 ceph-mon[82918]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:50:38.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:50:38.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:50:38.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:50:38.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:37 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:50:38.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:37 smithi172 ceph-mon[67636]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-02-07T12:50:38.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:37 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:38.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:50:38.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:38.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:38.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:38.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:50:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:50:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:50:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:50:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:50:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:50:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:50:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:50:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:50:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:50:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:50:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:50:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:50:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:50:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:50:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:50:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:50:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:50:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:50:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:50:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:50:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:50:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:50:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:50:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:38.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:38.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:38.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:38.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:38.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:38.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:38.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:37 smithi172 ceph-mon[67636]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1000 B/s rd, 0 op/s 2024-02-07T12:50:38.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:37 smithi172 ceph-mon[67636]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:50:38.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:50:38.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:50:38.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:50:39.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-mon[82918]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:50:39.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-mon[82918]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:50:39.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-mon[88137]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:50:39.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-mon[88137]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:50:39.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:50:38.995+0000 7fb71d671700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:39.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:50:39.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:39.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:39.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:39.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:39.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:39.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:50:39.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:39.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:50:39.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:39.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:50:39.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:39.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:39.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:39.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:39.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:39.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:39.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:39.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:39.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:39.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:50:39.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:50:39.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:50:39.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:50:39.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:50:39.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:50:39.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:50:39.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:50:39.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:50:39.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:50:39.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:50:39.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:50:39.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:50:39.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:50:39.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:50:39.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:50:39.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:50:39.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:39.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:50:39.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:39.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:39.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:39.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:39.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:39.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:50:39.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:39.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:50:39.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:39.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:50:39.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:39.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:39.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:39.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:39.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:39.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:39.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:39.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:39.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:38 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:39.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:38 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:50:39.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:38 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:50:40.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:39 smithi019 ceph-mon[82918]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 845 B/s rd, 0 op/s 2024-02-07T12:50:40.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:39 smithi019 ceph-mon[82918]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:40.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:50:40.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:40.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:40.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:40.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:40.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:40.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:50:40.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:40.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:50:40.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:40.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:50:40.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:40.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:40.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:40.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:40.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:40.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:40.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:40.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:40.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:40.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:50:40.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:50:40.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:50:40.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:50:40.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:50:40.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:50:40.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:50:40.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:50:40.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:50:40.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:50:40.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:50:40.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:50:40.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:50:40.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:50:40.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:50:40.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:50:40.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:50:40.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:40.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:50:40.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:40.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:40.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:40.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:40.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:40.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:50:40.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:40.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:50:40.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:40.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:50:40.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:40.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:40.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:40.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:40.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:40.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:40.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:40.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:40.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:40.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:39 smithi019 ceph-mon[88137]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 845 B/s rd, 0 op/s 2024-02-07T12:50:40.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:39 smithi019 ceph-mon[88137]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:40.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:50:40.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:40.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:40.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:40.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:40.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:40.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:50:40.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:40.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:50:40.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:40.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:50:40.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:40.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:40.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:40.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:40.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:40.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:40.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:40.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:40.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:40.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:50:40.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:50:40.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:50:40.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:50:40.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:50:40.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:50:40.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:50:40.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:50:40.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:50:40.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:50:40.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:50:40.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:50:40.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:50:40.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:50:40.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:50:40.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:50:40.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:50:40.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:40.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:50:40.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:40.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:40.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:40.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:40.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:40.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:50:40.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:40.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:50:40.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:40.355 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:50:40.355 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:40.355 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:40.355 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:40.355 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:40.355 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:40.355 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:40.355 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:40.355 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:40.356 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:40.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:39 smithi172 ceph-mon[67636]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 845 B/s rd, 0 op/s 2024-02-07T12:50:40.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:39 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:40.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:50:40.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:40.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:40.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:40.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:40.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:40.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:50:40.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:40.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:50:40.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:40.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:50:40.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:40.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:40.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:40.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:40.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:40.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:40.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:40.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:40.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:40.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:50:40.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:50:40.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:50:40.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:50:40.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:50:40.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:50:40.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:50:40.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:50:40.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:50:40.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:50:40.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:50:40.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:50:40.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:50:40.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:50:40.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:50:40.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:50:40.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:50:40.374 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:50:40.374 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:50:40.374 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:50:40.374 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:50:40.374 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:50:40.374 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:40.374 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:40.374 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:50:40.374 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:50:40.374 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:50:40.374 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:50:40.374 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:50:40.374 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:50:40.375 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:50:40.375 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:50:40.375 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:50:40.375 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:50:40.375 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:50:40.375 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:50:40.375 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:50:40.375 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:50:41.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:41 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:41.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:41 smithi019 ceph-mon[82918]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 843 B/s rd, 0 op/s 2024-02-07T12:50:41.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:41 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:41.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:41 smithi019 ceph-mon[88137]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 843 B/s rd, 0 op/s 2024-02-07T12:50:41.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:41 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:41.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:41 smithi172 ceph-mon[67636]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 843 B/s rd, 0 op/s 2024-02-07T12:50:44.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:43 smithi019 ceph-mon[82918]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:50:44.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:50:43] "GET /metrics HTTP/1.1" 200 38172 "" "Prometheus/2.43.0" 2024-02-07T12:50:44.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:43 smithi019 ceph-mon[88137]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:50:44.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:43 smithi172 ceph-mon[67636]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:50:46.171 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:45 smithi172 ceph-mon[67636]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 634 B/s rd, 0 op/s 2024-02-07T12:50:46.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:45 smithi019 ceph-mon[82918]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 634 B/s rd, 0 op/s 2024-02-07T12:50:46.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:45 smithi019 ceph-mon[88137]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 634 B/s rd, 0 op/s 2024-02-07T12:50:48.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:47 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:50:48.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:47 smithi019 ceph-mon[88137]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:50:48.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:47 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:50:48.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:47 smithi019 ceph-mon[82918]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:50:48.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:47 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:50:48.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:47 smithi172 ceph-mon[67636]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:50:50.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:49 smithi019 ceph-mon[82918]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:50:50.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:49 smithi019 ceph-mon[88137]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:50:50.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:49 smithi172 ceph-mon[67636]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:50:51.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:51 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:51.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:51 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:50:51.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:51 smithi019 ceph-mon[82918]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:50:51.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:51 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:51.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:51 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:50:51.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:51 smithi019 ceph-mon[88137]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:50:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:51 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:50:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:51 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:50:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:51 smithi172 ceph-mon[67636]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:50:53.722 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch ps' 2024-02-07T12:50:53.949 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:50:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:50:53] "GET /metrics HTTP/1.1" 200 38177 "" "Prometheus/2.43.0" 2024-02-07T12:50:54.241 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:53 smithi019 ceph-mon[88137]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:50:54.242 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:53 smithi019 ceph-mon[82918]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:50:54.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:53 smithi172 ceph-mon[67636]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:50:54.970 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:54 smithi019 ceph-mon[88137]: from='client.44316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:50:54.970 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:54 smithi019 ceph-mon[82918]: from='client.44316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:50:55.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:54 smithi172 ceph-mon[67636]: from='client.44316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:50:55.743 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:50:55.743 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (5m) 27s ago 27m 27.9M - 0.25.0 c8568f914cd2 fec9255f35e7 2024-02-07T12:50:55.743 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (4m) 2m ago 26m 82.9M - 9.4.12 a0ea5e3c46d9 aaffac63bc28 2024-02-07T12:50:55.743 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 unknown 27s ago 26m 63.4M - 3.5 e1d6a67b021e 5113aa77e3f2 2024-02-07T12:50:55.743 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283,8765 running (6m) 2m ago 31m 447M - 19.0.0-1252-g44458cce 79957b372307 5aa0c04fdfd7 2024-02-07T12:50:55.743 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (16m) 27s ago 33m 504M - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:50:55.743 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (3m) 27s ago 33m 49.4M 2048M 19.0.0-1252-g44458cce 79957b372307 8b1b0e21b1c5 2024-02-07T12:50:55.743 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (3m) 2m ago 31m 37.0M 2048M 19.0.0-1252-g44458cce 79957b372307 00f454bbac01 2024-02-07T12:50:55.744 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (2m) 27s ago 31m 39.9M 2048M 19.0.0-1252-g44458cce 79957b372307 6214003f6e08 2024-02-07T12:50:55.744 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (5m) 27s ago 27m 15.7M - 1.5.0 0da6a335fe13 c4bdcb1c4d49 2024-02-07T12:50:55.744 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (5m) 2m ago 27m 16.3M - 1.5.0 0da6a335fe13 04ea81f79d85 2024-02-07T12:50:55.744 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (53s) 27s ago 31m 53.7M 2993M 19.0.0-1252-g44458cce 79957b372307 78eddd34a903 2024-02-07T12:50:55.744 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (32s) 27s ago 30m 12.2M 2993M 19.0.0-1252-g44458cce 79957b372307 c37447e76c38 2024-02-07T12:50:55.744 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (30m) 27s ago 30m 54.8M 2993M 17.2.0 e1d6a67b021e 7db94f25a1aa 2024-02-07T12:50:55.744 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (30m) 27s ago 30m 55.4M 2993M 17.2.0 e1d6a67b021e 0b6b674188ed 2024-02-07T12:50:55.744 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (29m) 2m ago 29m 55.7M 3248M 17.2.0 e1d6a67b021e df72cfd69b3b 2024-02-07T12:50:55.744 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (29m) 2m ago 29m 52.4M 3248M 17.2.0 e1d6a67b021e 3bdb22c07537 2024-02-07T12:50:55.744 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (28m) 2m ago 28m 52.1M 3248M 17.2.0 e1d6a67b021e 7859b651e0d9 2024-02-07T12:50:55.745 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (28m) 2m ago 28m 55.5M 3248M 17.2.0 e1d6a67b021e b301e7c75d73 2024-02-07T12:50:55.745 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (5m) 2m ago 27m 61.0M - 2.43.0 a07b618ecd1d c9b607eea1be 2024-02-07T12:50:55.745 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (26m) 27s ago 26m 79.4M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:50:55.745 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (26m) 2m ago 26m 78.3M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:50:56.061 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:55 smithi019 ceph-mon[88137]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:50:56.061 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:55 smithi019 ceph-mon[82918]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:50:56.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:55 smithi172 ceph-mon[67636]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:50:56.376 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-02-07T12:50:57.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:56 smithi019 ceph-mon[88137]: from='client.54259 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:50:57.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:56 smithi019 ceph-mon[82918]: from='client.54259 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:50:57.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:56 smithi172 ceph-mon[67636]: from='client.54259 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:50:58.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:57 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:50:58.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:57 smithi019 ceph-mon[88137]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:50:58.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:57 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:50:58.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:57 smithi019 ceph-mon[82918]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:50:58.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:57 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:50:58.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:57 smithi172 ceph-mon[67636]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:50:58.486 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:50:59.075 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:58 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/3456540415' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:59.075 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:58 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/3456540415' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:50:59.099 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 7'"'"'' 2024-02-07T12:50:59.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:58 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/3456540415' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:00.331 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:50:59 smithi019 ceph-mon[82918]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:51:00.332 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:50:59 smithi019 ceph-mon[88137]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:51:00.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:50:59 smithi172 ceph-mon[67636]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:51:01.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:00 smithi019 ceph-mon[88137]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:51:01.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:00 smithi019 ceph-mon[82918]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:51:01.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:00 smithi172 ceph-mon[67636]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:51:02.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:01 smithi019 ceph-mon[82918]: from='client.44334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:02.247 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:01 smithi019 ceph-mon[88137]: from='client.44334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:02.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:01 smithi172 ceph-mon[67636]: from='client.44334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:03.095 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:51:03.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:02 smithi019 ceph-mon[82918]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:51:03.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:02 smithi019 ceph-mon[88137]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:51:03.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:02 smithi172 ceph-mon[67636]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:51:03.718 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch upgrade status' 2024-02-07T12:51:04.033 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:51:03] "GET /metrics HTTP/1.1" 200 38177 "" "Prometheus/2.43.0" 2024-02-07T12:51:05.616 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:51:05.616 INFO:teuthology.orchestra.run.smithi019.stdout: "target_image": null, 2024-02-07T12:51:05.616 INFO:teuthology.orchestra.run.smithi019.stdout: "in_progress": false, 2024-02-07T12:51:05.616 INFO:teuthology.orchestra.run.smithi019.stdout: "which": "", 2024-02-07T12:51:05.616 INFO:teuthology.orchestra.run.smithi019.stdout: "services_complete": [], 2024-02-07T12:51:05.616 INFO:teuthology.orchestra.run.smithi019.stdout: "progress": null, 2024-02-07T12:51:05.616 INFO:teuthology.orchestra.run.smithi019.stdout: "message": "", 2024-02-07T12:51:05.616 INFO:teuthology.orchestra.run.smithi019.stdout: "is_paused": false 2024-02-07T12:51:05.616 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:51:06.136 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:05 smithi019 ceph-mon[88137]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:51:06.136 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:51:06.136 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:05 smithi019 ceph-mon[82918]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:51:06.136 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:51:06.275 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph health detail' 2024-02-07T12:51:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:05 smithi172 ceph-mon[67636]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:51:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:51:07.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:06 smithi019 ceph-mon[88137]: from='client.44340 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:07.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:06 smithi019 ceph-mon[82918]: from='client.44340 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:07.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:06 smithi172 ceph-mon[67636]: from='client.44340 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:08.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:07 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:51:08.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:07 smithi019 ceph-mon[82918]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:51:08.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:07 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:51:08.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:07 smithi019 ceph-mon[88137]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:51:08.264 INFO:teuthology.orchestra.run.smithi019.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-02-07T12:51:08.264 INFO:teuthology.orchestra.run.smithi019.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-02-07T12:51:08.264 INFO:teuthology.orchestra.run.smithi019.stdout: daemon iscsi.foo.smithi019.svjimj on smithi019 is in unknown state 2024-02-07T12:51:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:07 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:51:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:07 smithi172 ceph-mon[67636]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:51:08.860 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1' 2024-02-07T12:51:09.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:08 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/1809485186' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:51:09.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:08 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/1809485186' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:51:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:08 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/1809485186' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:51:10.207 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:09 smithi019 ceph-mon[88137]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:51:10.207 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:09 smithi019 ceph-mon[82918]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:51:10.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:09 smithi172 ceph-mon[67636]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:51:12.022 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:11 smithi019 ceph-mon[88137]: from='client.54289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:12.022 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:11 smithi019 ceph-mon[88137]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:51:12.022 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:11 smithi019 ceph-mon[82918]: from='client.54289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:12.022 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:11 smithi019 ceph-mon[82918]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:51:12.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:11 smithi172 ceph-mon[67636]: from='client.54289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:12.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:11 smithi172 ceph-mon[67636]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:51:12.996 INFO:teuthology.orchestra.run.smithi019.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:51:13.573 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-02-07T12:51:14.188 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:51:13] "GET /metrics HTTP/1.1" 200 38170 "" "Prometheus/2.43.0" 2024-02-07T12:51:14.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:13 smithi019 ceph-mon[82918]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:51:14.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:13 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:14.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:13 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:51:14.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:13 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:51:14.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:13 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:51:14.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:13 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:14.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:13 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:51:14.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:13 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:51:14.189 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:13 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:51:14.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:13 smithi019 ceph-mon[88137]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:51:14.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:13 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:14.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:13 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:51:14.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:13 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:51:14.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:13 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:51:14.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:13 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:14.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:13 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:51:14.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:13 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:51:14.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:13 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:51:14.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:13 smithi172 ceph-mon[67636]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:51:14.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:13 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:14.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:13 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:51:14.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:13 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:51:14.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:13 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:51:14.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:13 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:14.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:13 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:51:14.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:13 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:51:14.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:13 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:51:15.202 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:14 smithi019 ceph-mon[82918]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:51:15.202 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:14 smithi019 ceph-mon[82918]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 959 B/s rd, 0 op/s 2024-02-07T12:51:15.203 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:14 smithi019 ceph-mon[82918]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:51:15.203 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:14 smithi019 ceph-mon[82918]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:51:15.203 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:14 smithi019 ceph-mon[82918]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:51:15.203 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:14 smithi019 ceph-mon[88137]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:51:15.203 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:14 smithi019 ceph-mon[88137]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 959 B/s rd, 0 op/s 2024-02-07T12:51:15.203 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:14 smithi019 ceph-mon[88137]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:51:15.203 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:14 smithi019 ceph-mon[88137]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:51:15.203 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:14 smithi019 ceph-mon[88137]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:51:15.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:14 smithi172 ceph-mon[67636]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:51:15.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:14 smithi172 ceph-mon[67636]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 959 B/s rd, 0 op/s 2024-02-07T12:51:15.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:14 smithi172 ceph-mon[67636]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:51:15.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:14 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:51:15.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:14 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:51:15.923 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:51:16.592 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:51:16.592 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (5m) 48s ago 27m 27.9M - 0.25.0 c8568f914cd2 fec9255f35e7 2024-02-07T12:51:16.592 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (5m) 3m ago 27m 82.9M - 9.4.12 a0ea5e3c46d9 aaffac63bc28 2024-02-07T12:51:16.592 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 unknown 48s ago 26m 63.4M - 3.5 e1d6a67b021e 5113aa77e3f2 2024-02-07T12:51:16.592 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283,8765 running (6m) 3m ago 31m 447M - 19.0.0-1252-g44458cce 79957b372307 5aa0c04fdfd7 2024-02-07T12:51:16.592 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (16m) 48s ago 33m 504M - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:51:16.592 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (3m) 48s ago 33m 49.4M 2048M 19.0.0-1252-g44458cce 79957b372307 8b1b0e21b1c5 2024-02-07T12:51:16.592 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (4m) 3m ago 32m 37.0M 2048M 19.0.0-1252-g44458cce 79957b372307 00f454bbac01 2024-02-07T12:51:16.592 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (2m) 48s ago 32m 39.9M 2048M 19.0.0-1252-g44458cce 79957b372307 6214003f6e08 2024-02-07T12:51:16.593 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (6m) 48s ago 28m 15.7M - 1.5.0 0da6a335fe13 c4bdcb1c4d49 2024-02-07T12:51:16.593 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (6m) 3m ago 28m 16.3M - 1.5.0 0da6a335fe13 04ea81f79d85 2024-02-07T12:51:16.593 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (74s) 48s ago 31m 53.7M 2993M 19.0.0-1252-g44458cce 79957b372307 78eddd34a903 2024-02-07T12:51:16.593 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (53s) 48s ago 31m 12.2M 2993M 19.0.0-1252-g44458cce 79957b372307 c37447e76c38 2024-02-07T12:51:16.593 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (30m) 48s ago 30m 54.8M 2993M 17.2.0 e1d6a67b021e 7db94f25a1aa 2024-02-07T12:51:16.593 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (30m) 48s ago 30m 55.4M 2993M 17.2.0 e1d6a67b021e 0b6b674188ed 2024-02-07T12:51:16.593 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (30m) 3m ago 30m 55.7M 3248M 17.2.0 e1d6a67b021e df72cfd69b3b 2024-02-07T12:51:16.593 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (29m) 3m ago 29m 52.4M 3248M 17.2.0 e1d6a67b021e 3bdb22c07537 2024-02-07T12:51:16.593 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (29m) 3m ago 29m 52.1M 3248M 17.2.0 e1d6a67b021e 7859b651e0d9 2024-02-07T12:51:16.593 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (28m) 3m ago 28m 55.5M 3248M 17.2.0 e1d6a67b021e b301e7c75d73 2024-02-07T12:51:16.593 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (5m) 3m ago 28m 61.0M - 2.43.0 a07b618ecd1d c9b607eea1be 2024-02-07T12:51:16.593 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (26m) 48s ago 26m 79.4M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:51:16.593 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (26m) 3m ago 26m 78.3M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:51:16.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:51:16.651+0000 7fb71d671700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:16.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:51:16.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:16.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:16.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:16.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:16.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:16.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:51:16.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:16.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:51:16.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:16.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:51:16.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:16.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:16.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:16.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:16.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:16.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:16.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:16.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:16.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:16.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:51:16.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:51:16.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:51:16.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:51:16.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:51:16.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:51:16.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:51:16.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:51:16.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:51:16.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:51:16.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:51:16.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:51:16.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:51:16.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:51:16.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:51:16.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:51:16.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:51:16.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:16.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:51:16.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:16.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:16.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:16.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:16.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:16.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:51:16.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:16.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:51:16.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:16.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:51:16.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:16.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:16.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:16.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:16.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:16.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:16.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:16.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:16.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:17.008 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:51:17.009 INFO:teuthology.orchestra.run.smithi019.stdout: "mon": { 2024-02-07T12:51:17.009 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 3 2024-02-07T12:51:17.009 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:51:17.009 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr": { 2024-02-07T12:51:17.009 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:51:17.009 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:51:17.009 INFO:teuthology.orchestra.run.smithi019.stdout: "osd": { 2024-02-07T12:51:17.009 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-02-07T12:51:17.009 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:51:17.009 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:51:17.009 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw": { 2024-02-07T12:51:17.009 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:51:17.009 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:51:17.009 INFO:teuthology.orchestra.run.smithi019.stdout: "overall": { 2024-02-07T12:51:17.009 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-02-07T12:51:17.010 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 7 2024-02-07T12:51:17.010 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:51:17.010 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:51:17.171 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:16 smithi172 ceph-mon[67636]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 590 B/s rd, 0 op/s 2024-02-07T12:51:17.171 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:16 smithi172 ceph-mon[67636]: from='client.54295 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:17.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-mon[82918]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 590 B/s rd, 0 op/s 2024-02-07T12:51:17.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-mon[82918]: from='client.54295 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:17.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-mon[88137]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 590 B/s rd, 0 op/s 2024-02-07T12:51:17.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:16 smithi019 ceph-mon[88137]: from='client.54295 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:17.364 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:51:17.364 INFO:teuthology.orchestra.run.smithi019.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", 2024-02-07T12:51:17.364 INFO:teuthology.orchestra.run.smithi019.stdout: "in_progress": true, 2024-02-07T12:51:17.364 INFO:teuthology.orchestra.run.smithi019.stdout: "which": "Upgrading daemons of type(s) crash,osd. Upgrade limited to 1 daemons (1 remaining).", 2024-02-07T12:51:17.364 INFO:teuthology.orchestra.run.smithi019.stdout: "services_complete": [], 2024-02-07T12:51:17.364 INFO:teuthology.orchestra.run.smithi019.stdout: "progress": "", 2024-02-07T12:51:17.364 INFO:teuthology.orchestra.run.smithi019.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 image", 2024-02-07T12:51:17.364 INFO:teuthology.orchestra.run.smithi019.stdout: "is_paused": false 2024-02-07T12:51:17.364 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:51:18.332 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:17 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:51:18.332 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:17 smithi019 ceph-mon[82918]: from='client.54301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:18.332 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:17 smithi019 ceph-mon[82918]: from='client.44370 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:18.332 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:17 smithi019 ceph-mon[82918]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:18.332 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:51:18.332 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:18.333 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:18.333 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:18.333 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:18.333 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:18.333 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:51:18.333 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:18.333 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:51:18.333 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:18.333 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:51:18.333 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:18.333 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:18.333 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:18.333 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:18.333 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:18.334 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:18.334 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:18.334 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:18.334 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:18.334 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:51:18.334 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:51:18.334 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:51:18.334 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:51:18.334 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:51:18.334 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:51:18.334 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:51:18.334 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:51:18.334 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:51:18.334 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:51:18.334 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:51:18.334 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:51:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:51:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:51:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:51:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:51:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:51:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:51:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:51:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:51:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:18.335 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:51:18.336 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:18.336 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:18.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:17 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:51:18.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:17 smithi019 ceph-mon[88137]: from='client.54301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:18.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:17 smithi019 ceph-mon[88137]: from='client.44370 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:18.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:17 smithi019 ceph-mon[88137]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:18.336 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:51:18.336 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:18.336 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:18.336 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:18.336 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:18.337 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:18.337 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:51:18.337 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:18.337 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:51:18.337 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:18.337 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:51:18.337 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:18.337 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:18.337 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:18.337 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:18.337 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:18.337 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:18.337 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:18.337 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:18.337 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:18.337 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:51:18.337 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:51:18.338 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:51:18.338 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:51:18.338 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:51:18.338 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:51:18.338 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:51:18.338 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:51:18.338 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:51:18.338 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:51:18.338 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:51:18.338 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:51:18.338 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:51:18.338 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:51:18.338 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:51:18.338 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:51:18.338 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:51:18.338 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:18.338 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:51:18.338 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:18.339 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:18.339 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:18.339 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:18.339 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:18.339 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:51:18.339 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:18.339 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:51:18.339 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:18.339 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:51:18.339 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:18.339 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:18.339 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:18.339 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:18.340 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:18.340 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:18.340 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:18.340 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:18.340 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:18.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:17 smithi019 ceph-mon[82918]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:51:18.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:17 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/1869947262' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:18.340 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:18.340 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:18.340 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:18.340 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:18.340 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:18.340 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:18.341 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:18.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:17 smithi019 ceph-mon[88137]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:51:18.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:17 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/1869947262' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:18.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:17 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:51:18.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:17 smithi172 ceph-mon[67636]: from='client.54301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:18.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:17 smithi172 ceph-mon[67636]: from='client.44370 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:18.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:17 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:18.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:51:18.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:18.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:18.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:18.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:18.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:18.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:51:18.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:18.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:51:18.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:18.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:51:18.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:18.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:18.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:18.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:18.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:18.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:18.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:18.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:18.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:18.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:51:18.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:51:18.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:51:18.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:51:18.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:51:18.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:51:18.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:51:18.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:51:18.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:51:18.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:51:18.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:51:18.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:51:18.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:51:18.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:51:18.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:51:18.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:51:18.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:51:18.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:18.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:51:18.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:18.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:18.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:18.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:18.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:18.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:51:18.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:18.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:51:18.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:18.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:51:18.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:18.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:18.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:18.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:18.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:18.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:18.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:18.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:18.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:18.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:17 smithi172 ceph-mon[67636]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:51:18.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:17 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/1869947262' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:19.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:18 smithi019 ceph-mon[82918]: from='client.54319 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:19.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:18 smithi019 ceph-mon[82918]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:51:19.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:18 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:19.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:18 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:51:19.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:18 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:19.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:18 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:19.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:18 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:19.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:18 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:19.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:18 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:19.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:18 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:19.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:18 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-02-07T12:51:19.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:18 smithi019 ceph-mon[88137]: from='client.54319 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:19.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:18 smithi019 ceph-mon[88137]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:51:19.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:18 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:19.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:18 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:51:19.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:18 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:19.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:18 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:19.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:18 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:19.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:18 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:19.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:18 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:19.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:18 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:19.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:18 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-02-07T12:51:19.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:18 smithi172 ceph-mon[67636]: from='client.54319 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:19.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:18 smithi172 ceph-mon[67636]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:51:19.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:18 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:19.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:18 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:51:19.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:18 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:19.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:18 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:19.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:18 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:19.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:18 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:19.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:18 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:19.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:18 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:19.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:18 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-02-07T12:51:20.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:19 smithi019 ceph-mon[82918]: Upgrade: Target is version 19.0.0-1252-g44458cce (squid) 2024-02-07T12:51:20.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:19 smithi019 ceph-mon[82918]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe'] 2024-02-07T12:51:20.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:19 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all mgr 2024-02-07T12:51:20.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:19 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all mon 2024-02-07T12:51:20.247 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:19 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all crash 2024-02-07T12:51:20.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:19 smithi019 ceph-mon[82918]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-02-07T12:51:20.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:19 smithi019 ceph-mon[82918]: Upgrade: osd.2 is safe to restart 2024-02-07T12:51:20.248 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:19 smithi019 ceph-mon[88137]: Upgrade: Target is version 19.0.0-1252-g44458cce (squid) 2024-02-07T12:51:20.248 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:19 smithi019 ceph-mon[88137]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe'] 2024-02-07T12:51:20.248 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:19 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all mgr 2024-02-07T12:51:20.248 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:19 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all mon 2024-02-07T12:51:20.248 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:19 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all crash 2024-02-07T12:51:20.248 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:19 smithi019 ceph-mon[88137]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-02-07T12:51:20.248 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:19 smithi019 ceph-mon[88137]: Upgrade: osd.2 is safe to restart 2024-02-07T12:51:20.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:19 smithi172 ceph-mon[67636]: Upgrade: Target is version 19.0.0-1252-g44458cce (squid) 2024-02-07T12:51:20.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:19 smithi172 ceph-mon[67636]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe'] 2024-02-07T12:51:20.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:19 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all mgr 2024-02-07T12:51:20.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:19 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all mon 2024-02-07T12:51:20.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:19 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all crash 2024-02-07T12:51:20.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:19 smithi172 ceph-mon[67636]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-02-07T12:51:20.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:19 smithi172 ceph-mon[67636]: Upgrade: osd.2 is safe to restart 2024-02-07T12:51:21.225 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:20 smithi019 ceph-mon[82918]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:51:21.226 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:20 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:51:21.226 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:20 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:21.226 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:20 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-02-07T12:51:21.226 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:20 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:51:21.226 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:20 smithi019 ceph-mon[88137]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:51:21.226 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:20 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:51:21.226 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:20 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:21.226 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:20 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-02-07T12:51:21.226 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:20 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:51:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:20 smithi172 ceph-mon[67636]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:51:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:20 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:51:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:20 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:20 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-02-07T12:51:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:20 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:51:22.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:21 smithi019 ceph-mon[82918]: Upgrade: Updating osd.2 2024-02-07T12:51:22.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:21 smithi019 ceph-mon[82918]: Deploying daemon osd.2 on smithi019 2024-02-07T12:51:22.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:21 smithi019 ceph-mon[88137]: Upgrade: Updating osd.2 2024-02-07T12:51:22.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:21 smithi019 ceph-mon[88137]: Deploying daemon osd.2 on smithi019 2024-02-07T12:51:22.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:21 smithi172 ceph-mon[67636]: Upgrade: Updating osd.2 2024-02-07T12:51:22.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:21 smithi172 ceph-mon[67636]: Deploying daemon osd.2 on smithi019 2024-02-07T12:51:23.294 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:22 smithi019 ceph-mon[88137]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:51:23.295 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:22 smithi019 ceph-mon[82918]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:51:23.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:22 smithi172 ceph-mon[67636]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:51:23.594 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:23 smithi019 systemd[1]: Stopping Ceph osd.2 for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:51:24.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:23 smithi019 ceph-mon[82918]: osd.2 marked itself down and dead 2024-02-07T12:51:24.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:23 smithi019 ceph-mon[88137]: osd.2 marked itself down and dead 2024-02-07T12:51:24.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:51:23] "GET /metrics HTTP/1.1" 200 38175 "" "Prometheus/2.43.0" 2024-02-07T12:51:24.095 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2[38018]: 2024-02-07T12:51:23.740+0000 7f54d7e79700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:51:24.095 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2[38018]: 2024-02-07T12:51:23.740+0000 7f54d7e79700 -1 osd.2 115 *** Got signal Terminated *** 2024-02-07T12:51:24.095 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2[38018]: 2024-02-07T12:51:23.740+0000 7f54d7e79700 -1 osd.2 115 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:51:24.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:23 smithi172 ceph-mon[67636]: osd.2 marked itself down and dead 2024-02-07T12:51:24.717 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:24 smithi019 podman[107656]: 2024-02-07 12:51:24.414158368 +0000 UTC m=+0.792817554 container died 7db94f25a1aa2b17f9b34e2a8a1c5fc46d5157b6299d9f24ebd0a040bdda54c7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, release=754, ceph=True, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, architecture=x86_64, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, maintainer=Guillaume Abrioux , vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD) 2024-02-07T12:51:25.067 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:24 smithi019 podman[107656]: 2024-02-07 12:51:24.806674821 +0000 UTC m=+1.185334006 container cleanup 7db94f25a1aa2b17f9b34e2a8a1c5fc46d5157b6299d9f24ebd0a040bdda54c7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2, vcs-type=git, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, io.openshift.expose-services=, release=754, ceph=True, maintainer=Guillaume Abrioux , architecture=x86_64, GIT_BRANCH=HEAD, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, vendor=Red Hat, Inc., distribution-scope=public) 2024-02-07T12:51:25.068 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:24 smithi019 bash[107656]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2 2024-02-07T12:51:25.068 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:24 smithi019 ceph-mon[88137]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:51:25.068 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:24 smithi019 ceph-mon[88137]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:51:25.068 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:24 smithi019 ceph-mon[88137]: failed to encode map e116 with expected crc 2024-02-07T12:51:25.068 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:24 smithi019 ceph-mon[88137]: failed to encode map e116 with expected crc 2024-02-07T12:51:25.068 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:24 smithi019 ceph-mon[88137]: failed to encode map e116 with expected crc 2024-02-07T12:51:25.068 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:24 smithi019 ceph-mon[88137]: osdmap e116: 8 total, 7 up, 8 in 2024-02-07T12:51:25.069 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:24 smithi019 ceph-mon[88137]: failed to encode map e116 with expected crc 2024-02-07T12:51:25.069 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:24 smithi019 ceph-mon[88137]: failed to encode map e116 with expected crc 2024-02-07T12:51:25.069 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:24 smithi019 ceph-mon[88137]: failed to encode map e116 with expected crc 2024-02-07T12:51:25.069 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:24 smithi019 ceph-mon[88137]: failed to encode map e116 with expected crc 2024-02-07T12:51:25.069 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:24 smithi019 ceph-mon[82918]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:51:25.069 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:24 smithi019 ceph-mon[82918]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:51:25.069 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:24 smithi019 ceph-mon[82918]: failed to encode map e116 with expected crc 2024-02-07T12:51:25.069 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:24 smithi019 ceph-mon[82918]: failed to encode map e116 with expected crc 2024-02-07T12:51:25.069 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:24 smithi019 ceph-mon[82918]: failed to encode map e116 with expected crc 2024-02-07T12:51:25.069 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:24 smithi019 ceph-mon[82918]: osdmap e116: 8 total, 7 up, 8 in 2024-02-07T12:51:25.069 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:24 smithi019 ceph-mon[82918]: failed to encode map e116 with expected crc 2024-02-07T12:51:25.069 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:24 smithi019 ceph-mon[82918]: failed to encode map e116 with expected crc 2024-02-07T12:51:25.069 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:24 smithi019 ceph-mon[82918]: failed to encode map e116 with expected crc 2024-02-07T12:51:25.070 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:24 smithi019 ceph-mon[82918]: failed to encode map e116 with expected crc 2024-02-07T12:51:25.344 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:25 smithi019 podman[107673]: 2024-02-07 12:51:25.06635203 +0000 UTC m=+0.649937416 container remove 7db94f25a1aa2b17f9b34e2a8a1c5fc46d5157b6299d9f24ebd0a040bdda54c7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2, architecture=x86_64, vendor=Red Hat, Inc., RELEASE=HEAD, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, release=754, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git) 2024-02-07T12:51:25.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:24 smithi172 ceph-mon[67636]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:51:25.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:24 smithi172 ceph-mon[67636]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:51:25.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:24 smithi172 ceph-mon[67636]: failed to encode map e116 with expected crc 2024-02-07T12:51:25.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:24 smithi172 ceph-mon[67636]: failed to encode map e116 with expected crc 2024-02-07T12:51:25.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:24 smithi172 ceph-mon[67636]: failed to encode map e116 with expected crc 2024-02-07T12:51:25.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:24 smithi172 ceph-mon[67636]: osdmap e116: 8 total, 7 up, 8 in 2024-02-07T12:51:25.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:24 smithi172 ceph-mon[67636]: failed to encode map e116 with expected crc 2024-02-07T12:51:25.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:24 smithi172 ceph-mon[67636]: failed to encode map e116 with expected crc 2024-02-07T12:51:25.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:24 smithi172 ceph-mon[67636]: failed to encode map e116 with expected crc 2024-02-07T12:51:25.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:24 smithi172 ceph-mon[67636]: failed to encode map e116 with expected crc 2024-02-07T12:51:25.680 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:25 smithi019 podman[107766]: 2024-02-07 12:51:25.283070972 +0000 UTC m=+0.017697104 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:51:25.680 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:25 smithi019 podman[107766]: 2024-02-07 12:51:25.392396149 +0000 UTC m=+0.127022316 container create 04d8208c9e5f2bd58024a1fae0f5c910ab985c19ff067a3b4d39a7f7153996fb (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2-deactivate, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:51:25.680 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:25 smithi019 podman[107766]: 2024-02-07 12:51:25.595136886 +0000 UTC m=+0.329763026 container init 04d8208c9e5f2bd58024a1fae0f5c910ab985c19ff067a3b4d39a7f7153996fb (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:51:25.680 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:25 smithi019 podman[107766]: 2024-02-07 12:51:25.598513317 +0000 UTC m=+0.333139443 container start 04d8208c9e5f2bd58024a1fae0f5c910ab985c19ff067a3b4d39a7f7153996fb (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240131, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:51:25.680 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:25 smithi019 podman[107766]: 2024-02-07 12:51:25.678443383 +0000 UTC m=+0.413069511 container attach 04d8208c9e5f2bd58024a1fae0f5c910ab985c19ff067a3b4d39a7f7153996fb (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, CEPH_POINT_RELEASE=) 2024-02-07T12:51:25.958 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:25 smithi019 podman[107766]: 2024-02-07 12:51:25.898784922 +0000 UTC m=+0.633411059 container died 04d8208c9e5f2bd58024a1fae0f5c910ab985c19ff067a3b4d39a7f7153996fb (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240131, CEPH_POINT_RELEASE=, io.buildah.version=1.33.2) 2024-02-07T12:51:26.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:25 smithi019 ceph-mon[88137]: failed to encode map e117 with expected crc 2024-02-07T12:51:26.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:25 smithi019 ceph-mon[88137]: failed to encode map e117 with expected crc 2024-02-07T12:51:26.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:25 smithi019 ceph-mon[88137]: osdmap e117: 8 total, 7 up, 8 in 2024-02-07T12:51:26.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:25 smithi019 ceph-mon[88137]: failed to encode map e117 with expected crc 2024-02-07T12:51:26.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:25 smithi019 ceph-mon[88137]: failed to encode map e117 with expected crc 2024-02-07T12:51:26.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:25 smithi019 ceph-mon[88137]: failed to encode map e117 with expected crc 2024-02-07T12:51:26.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:25 smithi019 ceph-mon[88137]: failed to encode map e117 with expected crc 2024-02-07T12:51:26.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:25 smithi019 ceph-mon[88137]: failed to encode map e117 with expected crc 2024-02-07T12:51:26.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:25 smithi019 ceph-mon[88137]: failed to encode map e117 with expected crc 2024-02-07T12:51:26.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:25 smithi019 ceph-mon[82918]: failed to encode map e117 with expected crc 2024-02-07T12:51:26.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:25 smithi019 ceph-mon[82918]: failed to encode map e117 with expected crc 2024-02-07T12:51:26.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:25 smithi019 ceph-mon[82918]: osdmap e117: 8 total, 7 up, 8 in 2024-02-07T12:51:26.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:25 smithi019 ceph-mon[82918]: failed to encode map e117 with expected crc 2024-02-07T12:51:26.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:25 smithi019 ceph-mon[82918]: failed to encode map e117 with expected crc 2024-02-07T12:51:26.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:25 smithi019 ceph-mon[82918]: failed to encode map e117 with expected crc 2024-02-07T12:51:26.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:25 smithi019 ceph-mon[82918]: failed to encode map e117 with expected crc 2024-02-07T12:51:26.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:25 smithi019 ceph-mon[82918]: failed to encode map e117 with expected crc 2024-02-07T12:51:26.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:25 smithi019 ceph-mon[82918]: failed to encode map e117 with expected crc 2024-02-07T12:51:26.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:25 smithi172 ceph-mon[67636]: failed to encode map e117 with expected crc 2024-02-07T12:51:26.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:25 smithi172 ceph-mon[67636]: failed to encode map e117 with expected crc 2024-02-07T12:51:26.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:25 smithi172 ceph-mon[67636]: osdmap e117: 8 total, 7 up, 8 in 2024-02-07T12:51:26.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:25 smithi172 ceph-mon[67636]: failed to encode map e117 with expected crc 2024-02-07T12:51:26.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:25 smithi172 ceph-mon[67636]: failed to encode map e117 with expected crc 2024-02-07T12:51:26.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:25 smithi172 ceph-mon[67636]: failed to encode map e117 with expected crc 2024-02-07T12:51:26.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:25 smithi172 ceph-mon[67636]: failed to encode map e117 with expected crc 2024-02-07T12:51:26.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:25 smithi172 ceph-mon[67636]: failed to encode map e117 with expected crc 2024-02-07T12:51:26.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:25 smithi172 ceph-mon[67636]: failed to encode map e117 with expected crc 2024-02-07T12:51:26.720 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:26 smithi019 podman[107792]: 2024-02-07 12:51:26.457205065 +0000 UTC m=+0.549680865 container remove 04d8208c9e5f2bd58024a1fae0f5c910ab985c19ff067a3b4d39a7f7153996fb (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-02-07T12:51:26.720 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:26 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.2.service: Deactivated successfully. 2024-02-07T12:51:26.720 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:26 smithi019 systemd[1]: Stopped Ceph osd.2 for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:51:26.720 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:26 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.2.service: Consumed 12.594s CPU time. 2024-02-07T12:51:26.720 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:26 smithi019 systemd[1]: Starting Ceph osd.2 for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:51:27.002 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:26 smithi019 ceph-mon[82918]: pgmap v130: 161 pgs: 15 stale+active+clean, 146 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-02-07T12:51:27.002 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:26 smithi019 ceph-mon[88137]: pgmap v130: 161 pgs: 15 stale+active+clean, 146 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-02-07T12:51:27.003 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:26 smithi019 podman[107912]: 2024-02-07 12:51:26.883661361 +0000 UTC m=+0.105782771 container create d445a104ee359a028fc766124cb0143d66cec0d48761444d035ae672876773dc (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2-activate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:51:27.003 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:26 smithi019 podman[107912]: 2024-02-07 12:51:26.796226246 +0000 UTC m=+0.018347619 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:51:27.344 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:27 smithi019 podman[107912]: 2024-02-07 12:51:27.119867609 +0000 UTC m=+0.341988988 container init d445a104ee359a028fc766124cb0143d66cec0d48761444d035ae672876773dc (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2-activate, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131, CEPH_POINT_RELEASE=, io.buildah.version=1.33.2) 2024-02-07T12:51:27.344 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:27 smithi019 podman[107912]: 2024-02-07 12:51:27.124053339 +0000 UTC m=+0.346174711 container start d445a104ee359a028fc766124cb0143d66cec0d48761444d035ae672876773dc (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2-activate, GIT_BRANCH=HEAD, io.buildah.version=1.33.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240131, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:51:27.344 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:27 smithi019 podman[107912]: 2024-02-07 12:51:27.219998916 +0000 UTC m=+0.442120293 container attach d445a104ee359a028fc766124cb0143d66cec0d48761444d035ae672876773dc (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2-activate, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240131, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-02-07T12:51:27.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:26 smithi172 ceph-mon[67636]: pgmap v130: 161 pgs: 15 stale+active+clean, 146 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-02-07T12:51:27.844 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:27 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2-activate[107927]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-02-07T12:51:27.844 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:27 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2-activate[107927]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-02-07T12:51:27.845 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:27 smithi019 bash[107912]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-02-07T12:51:27.845 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:27 smithi019 bash[107912]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-02-07T12:51:27.845 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:27 smithi019 bash[107912]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-02-07T12:51:27.845 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:27 smithi019 bash[107912]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-02-07T12:51:27.845 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:27 smithi019 bash[107912]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-02-07T12:51:27.845 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:27 smithi019 bash[107912]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-02-07T12:51:27.845 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:27 smithi019 bash[107912]: --> ceph-volume raw activate successful for osd ID: 2 2024-02-07T12:51:27.845 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:27 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2-activate[107927]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-02-07T12:51:27.845 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:27 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2-activate[107927]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-02-07T12:51:27.845 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:27 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2-activate[107927]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-02-07T12:51:27.845 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:27 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2-activate[107927]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-02-07T12:51:27.845 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:27 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2-activate[107927]: --> ceph-volume raw activate successful for osd ID: 2 2024-02-07T12:51:27.845 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:27 smithi019 podman[107912]: 2024-02-07 12:51:27.595072279 +0000 UTC m=+0.817193644 container died d445a104ee359a028fc766124cb0143d66cec0d48761444d035ae672876773dc (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2-activate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:51:28.183 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:27 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:51:28.183 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:27 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:51:28.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:27 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:51:28.506 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:28 smithi019 podman[108046]: 2024-02-07 12:51:28.18222444 +0000 UTC m=+0.581856767 container remove d445a104ee359a028fc766124cb0143d66cec0d48761444d035ae672876773dc (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2-activate, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-02-07T12:51:28.507 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:28 smithi019 podman[108109]: 2024-02-07 12:51:28.295505823 +0000 UTC m=+0.017788062 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:51:28.507 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:28 smithi019 podman[108109]: 2024-02-07 12:51:28.408237714 +0000 UTC m=+0.130519985 container create 90352897da859ee60b2030e764b1a4b756eeaa800a6daf3a66b77a9f6a5780b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240131, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True) 2024-02-07T12:51:28.762 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:28 smithi019 podman[108109]: 2024-02-07 12:51:28.653128393 +0000 UTC m=+0.375410640 container init 90352897da859ee60b2030e764b1a4b756eeaa800a6daf3a66b77a9f6a5780b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux ) 2024-02-07T12:51:28.762 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:28 smithi019 podman[108109]: 2024-02-07 12:51:28.656227812 +0000 UTC m=+0.378510050 container start 90352897da859ee60b2030e764b1a4b756eeaa800a6daf3a66b77a9f6a5780b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:51:28.762 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:28 smithi019 ceph-osd[108128]: -- 172.21.15.19:0/2109999491 <== mon.1 v2:172.21.15.19:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d31cc756c0 con 0x55d31cc47000 2024-02-07T12:51:29.094 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:28 smithi019 bash[108109]: 90352897da859ee60b2030e764b1a4b756eeaa800a6daf3a66b77a9f6a5780b4 2024-02-07T12:51:29.094 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:28 smithi019 systemd[1]: Started Ceph osd.2 for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:51:29.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:28 smithi019 ceph-mon[82918]: pgmap v131: 161 pgs: 13 active+undersized+degraded, 1 stale+active+clean, 28 active+undersized, 119 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 38/627 objects degraded (6.061%) 2024-02-07T12:51:29.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:28 smithi019 ceph-mon[82918]: Health check failed: Degraded data redundancy: 38/627 objects degraded (6.061%), 13 pgs degraded (PG_DEGRADED) 2024-02-07T12:51:29.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:28 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:29.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:28 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:29.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:28 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:29.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:28 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:51:29.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:28 smithi019 ceph-mon[88137]: pgmap v131: 161 pgs: 13 active+undersized+degraded, 1 stale+active+clean, 28 active+undersized, 119 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 38/627 objects degraded (6.061%) 2024-02-07T12:51:29.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:28 smithi019 ceph-mon[88137]: Health check failed: Degraded data redundancy: 38/627 objects degraded (6.061%), 13 pgs degraded (PG_DEGRADED) 2024-02-07T12:51:29.096 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:28 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:29.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:28 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:29.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:28 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:29.097 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:28 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:51:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:28 smithi172 ceph-mon[67636]: pgmap v131: 161 pgs: 13 active+undersized+degraded, 1 stale+active+clean, 28 active+undersized, 119 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 38/627 objects degraded (6.061%) 2024-02-07T12:51:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:28 smithi172 ceph-mon[67636]: Health check failed: Degraded data redundancy: 38/627 objects degraded (6.061%), 13 pgs degraded (PG_DEGRADED) 2024-02-07T12:51:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:28 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:28 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:28 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:28 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:51:29.594 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:29 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2[108124]: 2024-02-07T12:51:29.199+0000 7f15510c17c0 -1 Falling back to public interface 2024-02-07T12:51:31.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:30 smithi019 ceph-mon[82918]: pgmap v132: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 57/627 objects degraded (9.091%) 2024-02-07T12:51:31.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:30 smithi019 ceph-mon[88137]: pgmap v132: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 57/627 objects degraded (9.091%) 2024-02-07T12:51:31.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:30 smithi172 ceph-mon[67636]: pgmap v132: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 57/627 objects degraded (9.091%) 2024-02-07T12:51:31.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:31 smithi019 ceph-mon[82918]: from='osd.2 [v2:172.21.15.19:6818/1920561812,v1:172.21.15.19:6819/1920561812]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-02-07T12:51:31.992 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:31 smithi019 ceph-mon[82918]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-02-07T12:51:31.993 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:31 smithi019 ceph-mon[88137]: from='osd.2 [v2:172.21.15.19:6818/1920561812,v1:172.21.15.19:6819/1920561812]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-02-07T12:51:31.993 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:31 smithi019 ceph-mon[88137]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-02-07T12:51:31.993 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:31 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2[108124]: 2024-02-07T12:51:31.920+0000 7f15510c17c0 -1 osd.2 0 read_superblock omap replica is missing. 2024-02-07T12:51:31.993 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:31 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2[108124]: 2024-02-07T12:51:31.941+0000 7f15510c17c0 -1 osd.2 115 log_to_monitors true 2024-02-07T12:51:32.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:31 smithi172 ceph-mon[67636]: from='osd.2 [v2:172.21.15.19:6818/1920561812,v1:172.21.15.19:6819/1920561812]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-02-07T12:51:32.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:31 smithi172 ceph-mon[67636]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-02-07T12:51:32.844 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:51:32 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2[108124]: 2024-02-07T12:51:32.443+0000 7f153e518700 -1 osd.2 115 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:51:33.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:32 smithi019 ceph-mon[88137]: pgmap v133: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 57/627 objects degraded (9.091%) 2024-02-07T12:51:33.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:32 smithi019 ceph-mon[88137]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-02-07T12:51:33.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:32 smithi019 ceph-mon[88137]: failed to encode map e118 with expected crc 2024-02-07T12:51:33.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:32 smithi019 ceph-mon[88137]: failed to encode map e118 with expected crc 2024-02-07T12:51:33.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:32 smithi019 ceph-mon[88137]: osdmap e118: 8 total, 7 up, 8 in 2024-02-07T12:51:33.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:32 smithi019 ceph-mon[88137]: from='osd.2 [v2:172.21.15.19:6818/1920561812,v1:172.21.15.19:6819/1920561812]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:51:33.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:32 smithi019 ceph-mon[88137]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:51:33.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:32 smithi019 ceph-mon[88137]: failed to encode map e118 with expected crc 2024-02-07T12:51:33.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:32 smithi019 ceph-mon[88137]: failed to encode map e118 with expected crc 2024-02-07T12:51:33.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:32 smithi019 ceph-mon[88137]: from='osd.2 ' entity='osd.2' 2024-02-07T12:51:33.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:32 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:33.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:32 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:33.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:32 smithi019 ceph-mon[82918]: pgmap v133: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 57/627 objects degraded (9.091%) 2024-02-07T12:51:33.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:32 smithi019 ceph-mon[82918]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-02-07T12:51:33.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:32 smithi019 ceph-mon[82918]: failed to encode map e118 with expected crc 2024-02-07T12:51:33.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:32 smithi019 ceph-mon[82918]: failed to encode map e118 with expected crc 2024-02-07T12:51:33.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:32 smithi019 ceph-mon[82918]: osdmap e118: 8 total, 7 up, 8 in 2024-02-07T12:51:33.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:32 smithi019 ceph-mon[82918]: from='osd.2 [v2:172.21.15.19:6818/1920561812,v1:172.21.15.19:6819/1920561812]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:51:33.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:32 smithi019 ceph-mon[82918]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:51:33.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:32 smithi019 ceph-mon[82918]: failed to encode map e118 with expected crc 2024-02-07T12:51:33.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:32 smithi019 ceph-mon[82918]: failed to encode map e118 with expected crc 2024-02-07T12:51:33.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:32 smithi019 ceph-mon[82918]: from='osd.2 ' entity='osd.2' 2024-02-07T12:51:33.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:32 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:33.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:32 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:32 smithi172 ceph-mon[67636]: pgmap v133: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 57/627 objects degraded (9.091%) 2024-02-07T12:51:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:32 smithi172 ceph-mon[67636]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-02-07T12:51:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:32 smithi172 ceph-mon[67636]: failed to encode map e118 with expected crc 2024-02-07T12:51:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:32 smithi172 ceph-mon[67636]: failed to encode map e118 with expected crc 2024-02-07T12:51:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:32 smithi172 ceph-mon[67636]: osdmap e118: 8 total, 7 up, 8 in 2024-02-07T12:51:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:32 smithi172 ceph-mon[67636]: from='osd.2 [v2:172.21.15.19:6818/1920561812,v1:172.21.15.19:6819/1920561812]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:51:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:32 smithi172 ceph-mon[67636]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:51:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:32 smithi172 ceph-mon[67636]: failed to encode map e118 with expected crc 2024-02-07T12:51:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:32 smithi172 ceph-mon[67636]: failed to encode map e118 with expected crc 2024-02-07T12:51:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:32 smithi172 ceph-mon[67636]: from='osd.2 ' entity='osd.2' 2024-02-07T12:51:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:32 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:32 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:34.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:51:33] "GET /metrics HTTP/1.1" 200 38175 "" "Prometheus/2.43.0" 2024-02-07T12:51:34.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:33 smithi019 ceph-mon[88137]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:51:34.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:33 smithi019 ceph-mon[88137]: failed to encode map e119 with expected crc 2024-02-07T12:51:34.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:33 smithi019 ceph-mon[88137]: osd.2 [v2:172.21.15.19:6818/1920561812,v1:172.21.15.19:6819/1920561812] boot 2024-02-07T12:51:34.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:33 smithi019 ceph-mon[88137]: osdmap e119: 8 total, 8 up, 8 in 2024-02-07T12:51:34.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:33 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:51:34.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:33 smithi019 ceph-mon[88137]: pgmap v136: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 151 MiB used, 715 GiB / 715 GiB avail; 57/627 objects degraded (9.091%) 2024-02-07T12:51:34.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:33 smithi019 ceph-mon[82918]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:51:34.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:33 smithi019 ceph-mon[82918]: failed to encode map e119 with expected crc 2024-02-07T12:51:34.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:33 smithi019 ceph-mon[82918]: osd.2 [v2:172.21.15.19:6818/1920561812,v1:172.21.15.19:6819/1920561812] boot 2024-02-07T12:51:34.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:33 smithi019 ceph-mon[82918]: osdmap e119: 8 total, 8 up, 8 in 2024-02-07T12:51:34.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:33 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:51:34.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:33 smithi019 ceph-mon[82918]: pgmap v136: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 151 MiB used, 715 GiB / 715 GiB avail; 57/627 objects degraded (9.091%) 2024-02-07T12:51:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:33 smithi172 ceph-mon[67636]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:51:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:33 smithi172 ceph-mon[67636]: failed to encode map e119 with expected crc 2024-02-07T12:51:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:33 smithi172 ceph-mon[67636]: osd.2 [v2:172.21.15.19:6818/1920561812,v1:172.21.15.19:6819/1920561812] boot 2024-02-07T12:51:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:33 smithi172 ceph-mon[67636]: osdmap e119: 8 total, 8 up, 8 in 2024-02-07T12:51:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:33 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:51:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:33 smithi172 ceph-mon[67636]: pgmap v136: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 151 MiB used, 715 GiB / 715 GiB avail; 57/627 objects degraded (9.091%) 2024-02-07T12:51:35.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:35 smithi019 ceph-mon[82918]: failed to encode map e119 with expected crc 2024-02-07T12:51:35.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:35 smithi019 ceph-mon[82918]: failed to encode map e119 with expected crc 2024-02-07T12:51:35.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:35 smithi019 ceph-mon[82918]: failed to encode map e119 with expected crc 2024-02-07T12:51:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:35 smithi019 ceph-mon[82918]: failed to encode map e119 with expected crc 2024-02-07T12:51:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:35 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:35 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:35 smithi019 ceph-mon[82918]: failed to encode map e120 with expected crc 2024-02-07T12:51:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:35 smithi019 ceph-mon[82918]: osdmap e120: 8 total, 8 up, 8 in 2024-02-07T12:51:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:35 smithi019 ceph-mon[82918]: failed to encode map e120 with expected crc 2024-02-07T12:51:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:35 smithi019 ceph-mon[82918]: failed to encode map e120 with expected crc 2024-02-07T12:51:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:35 smithi019 ceph-mon[82918]: failed to encode map e120 with expected crc 2024-02-07T12:51:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:35 smithi019 ceph-mon[82918]: Health check update: Degraded data redundancy: 57/627 objects degraded (9.091%), 15 pgs degraded (PG_DEGRADED) 2024-02-07T12:51:35.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:35 smithi019 ceph-mon[88137]: failed to encode map e119 with expected crc 2024-02-07T12:51:35.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:35 smithi019 ceph-mon[88137]: failed to encode map e119 with expected crc 2024-02-07T12:51:35.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:35 smithi019 ceph-mon[88137]: failed to encode map e119 with expected crc 2024-02-07T12:51:35.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:35 smithi019 ceph-mon[88137]: failed to encode map e119 with expected crc 2024-02-07T12:51:35.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:35 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:35.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:35 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:35.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:35 smithi019 ceph-mon[88137]: failed to encode map e120 with expected crc 2024-02-07T12:51:35.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:35 smithi019 ceph-mon[88137]: osdmap e120: 8 total, 8 up, 8 in 2024-02-07T12:51:35.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:35 smithi019 ceph-mon[88137]: failed to encode map e120 with expected crc 2024-02-07T12:51:35.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:35 smithi019 ceph-mon[88137]: failed to encode map e120 with expected crc 2024-02-07T12:51:35.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:35 smithi019 ceph-mon[88137]: failed to encode map e120 with expected crc 2024-02-07T12:51:35.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:35 smithi019 ceph-mon[88137]: Health check update: Degraded data redundancy: 57/627 objects degraded (9.091%), 15 pgs degraded (PG_DEGRADED) 2024-02-07T12:51:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:35 smithi172 ceph-mon[67636]: failed to encode map e119 with expected crc 2024-02-07T12:51:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:35 smithi172 ceph-mon[67636]: failed to encode map e119 with expected crc 2024-02-07T12:51:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:35 smithi172 ceph-mon[67636]: failed to encode map e119 with expected crc 2024-02-07T12:51:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:35 smithi172 ceph-mon[67636]: failed to encode map e119 with expected crc 2024-02-07T12:51:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:35 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:35 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:35 smithi172 ceph-mon[67636]: failed to encode map e120 with expected crc 2024-02-07T12:51:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:35 smithi172 ceph-mon[67636]: osdmap e120: 8 total, 8 up, 8 in 2024-02-07T12:51:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:35 smithi172 ceph-mon[67636]: failed to encode map e120 with expected crc 2024-02-07T12:51:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:35 smithi172 ceph-mon[67636]: failed to encode map e120 with expected crc 2024-02-07T12:51:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:35 smithi172 ceph-mon[67636]: failed to encode map e120 with expected crc 2024-02-07T12:51:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:35 smithi172 ceph-mon[67636]: Health check update: Degraded data redundancy: 57/627 objects degraded (9.091%), 15 pgs degraded (PG_DEGRADED) 2024-02-07T12:51:36.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:36 smithi019 ceph-mon[82918]: failed to encode map e120 with expected crc 2024-02-07T12:51:36.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:36 smithi019 ceph-mon[82918]: failed to encode map e120 with expected crc 2024-02-07T12:51:36.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:36 smithi019 ceph-mon[82918]: failed to encode map e120 with expected crc 2024-02-07T12:51:36.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:36 smithi019 ceph-mon[82918]: failed to encode map e120 with expected crc 2024-02-07T12:51:36.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:36 smithi019 ceph-mon[82918]: failed to encode map e120 with expected crc 2024-02-07T12:51:36.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:36 smithi019 ceph-mon[82918]: failed to encode map e120 with expected crc 2024-02-07T12:51:36.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:36 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:36.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:36 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:51:36.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:36 smithi019 ceph-mon[82918]: pgmap v138: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 151 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 57/627 objects degraded (9.091%) 2024-02-07T12:51:36.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:36 smithi019 ceph-mon[88137]: failed to encode map e120 with expected crc 2024-02-07T12:51:36.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:36 smithi019 ceph-mon[88137]: failed to encode map e120 with expected crc 2024-02-07T12:51:36.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:36 smithi019 ceph-mon[88137]: failed to encode map e120 with expected crc 2024-02-07T12:51:36.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:36 smithi019 ceph-mon[88137]: failed to encode map e120 with expected crc 2024-02-07T12:51:36.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:36 smithi019 ceph-mon[88137]: failed to encode map e120 with expected crc 2024-02-07T12:51:36.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:36 smithi019 ceph-mon[88137]: failed to encode map e120 with expected crc 2024-02-07T12:51:36.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:36 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:36.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:36 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:51:36.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:36 smithi019 ceph-mon[88137]: pgmap v138: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 151 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 57/627 objects degraded (9.091%) 2024-02-07T12:51:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:36 smithi172 ceph-mon[67636]: failed to encode map e120 with expected crc 2024-02-07T12:51:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:36 smithi172 ceph-mon[67636]: failed to encode map e120 with expected crc 2024-02-07T12:51:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:36 smithi172 ceph-mon[67636]: failed to encode map e120 with expected crc 2024-02-07T12:51:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:36 smithi172 ceph-mon[67636]: failed to encode map e120 with expected crc 2024-02-07T12:51:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:36 smithi172 ceph-mon[67636]: failed to encode map e120 with expected crc 2024-02-07T12:51:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:36 smithi172 ceph-mon[67636]: failed to encode map e120 with expected crc 2024-02-07T12:51:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:36 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:36 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:51:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:36 smithi172 ceph-mon[67636]: pgmap v138: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 151 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 57/627 objects degraded (9.091%) 2024-02-07T12:51:37.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:37 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:51:37.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:37 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:51:37.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:37 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:51:38.798 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:38 smithi019 ceph-mon[88137]: pgmap v139: 161 pgs: 2 active+undersized+degraded, 3 active+undersized, 156 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 19/627 objects degraded (3.030%) 2024-02-07T12:51:38.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:38 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:38.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:38 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:38.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:38 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:51:38.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:38 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:51:38.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:38 smithi019 ceph-mon[88137]: pgmap v140: 161 pgs: 2 active+undersized+degraded, 3 active+undersized, 156 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 893 B/s rd, 0 op/s; 19/627 objects degraded (3.030%) 2024-02-07T12:51:38.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:38 smithi019 ceph-mon[88137]: pgmap v141: 161 pgs: 2 active+undersized+degraded, 3 active+undersized, 156 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 19/627 objects degraded (3.030%) 2024-02-07T12:51:38.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:38 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:38.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:38 smithi019 ceph-mon[88137]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:51:38.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:38 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:51:38.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:38 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:51:38.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:38 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:51:38.799 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:38 smithi019 ceph-mon[88137]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:51:38.799 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:38 smithi019 ceph-mon[82918]: pgmap v139: 161 pgs: 2 active+undersized+degraded, 3 active+undersized, 156 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 19/627 objects degraded (3.030%) 2024-02-07T12:51:38.799 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:38 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:38.800 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:38 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:38.800 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:38 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:51:38.800 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:38 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:51:38.800 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:38 smithi019 ceph-mon[82918]: pgmap v140: 161 pgs: 2 active+undersized+degraded, 3 active+undersized, 156 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 893 B/s rd, 0 op/s; 19/627 objects degraded (3.030%) 2024-02-07T12:51:38.800 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:38 smithi019 ceph-mon[82918]: pgmap v141: 161 pgs: 2 active+undersized+degraded, 3 active+undersized, 156 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 19/627 objects degraded (3.030%) 2024-02-07T12:51:38.800 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:38 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:38.800 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:38 smithi019 ceph-mon[82918]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:51:38.800 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:38 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:51:38.800 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:38 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:51:38.800 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:38 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:51:38.800 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:38 smithi019 ceph-mon[82918]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:51:39.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:38 smithi172 ceph-mon[67636]: pgmap v139: 161 pgs: 2 active+undersized+degraded, 3 active+undersized, 156 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 19/627 objects degraded (3.030%) 2024-02-07T12:51:39.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:38 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:39.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:38 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:39.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:38 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:51:39.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:38 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:51:39.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:38 smithi172 ceph-mon[67636]: pgmap v140: 161 pgs: 2 active+undersized+degraded, 3 active+undersized, 156 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 893 B/s rd, 0 op/s; 19/627 objects degraded (3.030%) 2024-02-07T12:51:39.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:38 smithi172 ceph-mon[67636]: pgmap v141: 161 pgs: 2 active+undersized+degraded, 3 active+undersized, 156 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 19/627 objects degraded (3.030%) 2024-02-07T12:51:39.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:38 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:39.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:38 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:51:39.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:38 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:51:39.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:38 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:51:39.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:38 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:51:39.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:38 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:51:39.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:51:39.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:39.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:39.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:39.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:39.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:39.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:39.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:39.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:39.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:39.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:51:39.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:39.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:39.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:39.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:39.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:39.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:39.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:39.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:39.846 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:39.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:51:39.670+0000 7fb71d671700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:39.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:51:39.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:39.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:39.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:39.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:39.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:39.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:51:39.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:39.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:51:39.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:39.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:51:39.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:39.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:39.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:39.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:39.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:39.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:39.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:39.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:39.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:39.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:51:39.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:51:39.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:51:39.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:51:39.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:51:39.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:51:39.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:51:39.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:51:39.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:51:39.850 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:51:39.850 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:51:39.850 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:51:39.850 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:51:39.850 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:51:39.850 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:51:39.850 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:51:39.850 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:51:39.850 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:39.850 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:51:39.850 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:39.850 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:39.851 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:39.851 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:39.851 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:39.851 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:51:39.851 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:39.851 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:51:39.851 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:39.851 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:51:39.851 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:39.851 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:39.851 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:39.851 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:39.852 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:39.852 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:39.852 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:39.852 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:39.852 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:40.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:39 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:51:40.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:39 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:40.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:39 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:40.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:39 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:40.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:39 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:40.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:39 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:40.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:39 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:40.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:39 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:40.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:39 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:40.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:39 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:40.723 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:40.723 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:51:40.723 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:40.723 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:40.723 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:40.724 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:40.724 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:40.724 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:51:40.724 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:40.724 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:51:40.725 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:40.725 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:51:40.725 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:40.725 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:40.725 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:40.725 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:40.725 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:40.726 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:40.726 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:40.726 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:40.726 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:40.726 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:51:40.726 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:51:40.726 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:51:40.727 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:51:40.727 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:51:40.727 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:51:40.727 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:51:40.727 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:51:40.727 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:51:40.728 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:51:40.728 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:51:40.728 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:51:40.728 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:51:40.728 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:51:40.728 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:51:40.728 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:51:40.729 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:51:40.729 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:40.729 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:51:40.729 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:40.729 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:40.729 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:40.729 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:40.730 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:40.730 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:51:40.730 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:40.730 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:51:40.730 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:40.730 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:51:40.731 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:40.731 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:40.731 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:40.731 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:40.732 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:40.732 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:51:40.732 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:40.732 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:40.732 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:40.732 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:40.733 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:40.733 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:51:40.733 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:40.733 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:51:40.733 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:40.733 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:51:40.734 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:40.734 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:40.734 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:40.734 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:40.734 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:40.735 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:40.735 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:40.735 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:40.735 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:40.735 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:51:40.735 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:51:40.736 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:51:40.736 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:51:40.736 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:51:40.736 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:51:40.736 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:51:40.736 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:51:40.737 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:51:40.737 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:51:40.737 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:51:40.737 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:51:40.737 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:51:40.737 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:51:40.737 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:51:40.738 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:51:40.738 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:51:40.738 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:40.738 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:51:40.738 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:40.738 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:40.739 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:40.739 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:40.739 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:40.739 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:51:40.739 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:40.739 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:51:40.739 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:40.739 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:51:40.739 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:40.739 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:40.740 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:40.740 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:40.740 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:40.740 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:40.741 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:40.741 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:40.741 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:40.741 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all nvmeof 2024-02-07T12:51:40.741 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: Upgrade: Finalizing container_image settings 2024-02-07T12:51:40.741 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:51:40.741 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:51:40.741 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-02-07T12:51:40.741 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.741 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.741 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 834 B/s rd, 0 op/s 2024-02-07T12:51:40.741 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-02-07T12:51:40.741 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:51:40.742 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:51:40.742 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-02-07T12:51:40.742 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:51:40.742 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:51:40.742 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:51:40.742 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:51:40.742 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:51:40.743 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:51:40.743 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:40.743 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:40.744 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:40.744 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:40.744 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:40.744 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all nvmeof 2024-02-07T12:51:40.744 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: Upgrade: Finalizing container_image settings 2024-02-07T12:51:40.745 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:51:40.745 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:51:40.745 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-02-07T12:51:40.745 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.745 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.746 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 834 B/s rd, 0 op/s 2024-02-07T12:51:40.746 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-02-07T12:51:40.746 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:51:40.746 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:51:40.746 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-02-07T12:51:40.747 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:51:40.747 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:51:40.747 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:51:40.747 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:51:40.747 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:51:40.748 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:51:40.748 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:51:40.748 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:51:40.749 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.749 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.749 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:51:40.749 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:51:40.749 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:51:40.750 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:51:40.750 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:51:40.750 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:51:40.750 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:51:40.751 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:51:40.751 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-02-07T12:51:40.751 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.751 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.751 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.752 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.752 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.752 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.752 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.752 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.753 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.753 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.753 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.754 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:51:40.754 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:51:40.754 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.754 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.754 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:51:40.755 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:51:40.755 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:51:40.755 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:51:40.755 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:51:40.755 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:51:40.755 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:51:40.756 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:51:40.756 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-02-07T12:51:40.756 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.756 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.757 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.757 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.757 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.757 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.757 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.758 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.758 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.758 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.758 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.759 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.759 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: Upgrade: Complete! 2024-02-07T12:51:40.759 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:51:40.759 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:51:40.760 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-02-07T12:51:40.760 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:51:40.760 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:51:40.760 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:51:40.760 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 963 B/s rd, 0 op/s 2024-02-07T12:51:40.761 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:51:40.761 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:40.761 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:51:40.761 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:51:40.761 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:51:40.762 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:51:40.762 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:51:40.762 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: Health check update: Degraded data redundancy: 19/627 objects degraded (3.030%), 2 pgs degraded (PG_DEGRADED) 2024-02-07T12:51:40.762 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:40.762 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:40.763 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: Upgrade: Complete! 2024-02-07T12:51:40.763 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:51:40.763 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:51:40.763 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-02-07T12:51:40.763 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:51:40.763 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:51:40.763 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:51:40.764 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 963 B/s rd, 0 op/s 2024-02-07T12:51:40.764 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:51:40.764 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:40.764 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:51:40.764 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:51:40.764 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:51:40.764 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:51:40.764 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:51:40.764 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: Health check update: Degraded data redundancy: 19/627 objects degraded (3.030%), 2 pgs degraded (PG_DEGRADED) 2024-02-07T12:51:40.765 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:41.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:41.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:51:41.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:41.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:41.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:41.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:41.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:41.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:51:41.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:41.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:51:41.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:41.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:51:41.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:41.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:41.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:41.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:41.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:41.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:41.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:41.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:41.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:41.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:51:41.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:51:41.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:51:41.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:51:41.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:51:41.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:51:41.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:51:41.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:51:41.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:51:41.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:51:41.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:51:41.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:51:41.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:51:41.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:51:41.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:51:41.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:51:41.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:51:41.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:41.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:51:41.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:41.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:41.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:41.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:41.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:41.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:51:41.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:41.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:51:41.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:41.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:51:41.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:41.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:41.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:41.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:41.124 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:41.124 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:41.124 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:41.124 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:41.124 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:41.124 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all nvmeof 2024-02-07T12:51:41.124 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: Upgrade: Finalizing container_image settings 2024-02-07T12:51:41.124 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:51:41.124 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:51:41.124 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-02-07T12:51:41.124 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:41.124 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:41.124 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 834 B/s rd, 0 op/s 2024-02-07T12:51:41.124 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-02-07T12:51:41.125 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:51:41.125 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:51:41.125 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-02-07T12:51:41.125 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:51:41.125 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:51:41.125 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:51:41.125 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:51:41.125 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:51:41.125 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:51:41.125 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:51:41.125 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:51:41.125 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:41.125 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:41.125 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:51:41.125 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:51:41.125 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:51:41.126 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:51:41.126 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:51:41.126 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:51:41.126 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:51:41.126 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:51:41.126 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-02-07T12:51:41.126 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:41.126 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:41.126 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:41.126 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:41.126 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:41.126 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:41.126 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:41.126 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:41.126 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:41.126 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:41.127 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:41.127 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:51:41.127 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: Upgrade: Complete! 2024-02-07T12:51:41.127 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:51:41.127 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:51:41.127 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-02-07T12:51:41.127 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:51:41.127 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:51:41.127 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:51:41.127 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 963 B/s rd, 0 op/s 2024-02-07T12:51:41.127 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:51:41.127 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:41.127 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:51:41.127 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:51:41.127 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:51:41.128 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:51:41.128 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:51:41.128 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: Health check update: Degraded data redundancy: 19/627 objects degraded (3.030%), 2 pgs degraded (PG_DEGRADED) 2024-02-07T12:51:41.128 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:41.704 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-mon[82918]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 19/627 objects degraded (3.030%), 2 pgs degraded) 2024-02-07T12:51:41.704 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:51:41.530+0000 7fb71d671700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:41.705 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:51:41.705 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:41.705 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:41.705 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:41.705 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:41.705 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:41.706 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:51:41.706 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:41.706 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:51:41.706 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:41.706 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:51:41.706 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:41.706 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:41.706 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:41.707 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:41.707 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:41.707 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:41.707 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:41.708 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:41.708 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:41.708 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:51:41.708 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:51:41.708 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:51:41.708 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:51:41.708 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:51:41.709 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:51:41.709 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:51:41.709 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:51:41.709 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:51:41.709 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:51:41.709 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:51:41.709 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:51:41.709 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:51:41.710 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:51:41.710 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:51:41.710 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:51:41.710 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:51:41.710 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:41.710 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:51:41.710 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:41.710 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:41.710 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:41.711 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:41.711 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:41.711 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:51:41.711 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:41.711 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:51:41.711 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:41.711 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:51:41.711 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:41.711 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:41.712 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:41.712 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:41.712 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:41.712 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:41.712 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:41.712 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:41.712 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:42.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-mon[88137]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 19/627 objects degraded (3.030%), 2 pgs degraded) 2024-02-07T12:51:42.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:51:42.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:51:42.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:51:42.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:42.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:51:42.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:51:42.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:51:42.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:51:42.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:51:42.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:51:42.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:42.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:51:42.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:51:42.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:41 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:51:42.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:41 smithi172 ceph-mon[67636]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 19/627 objects degraded (3.030%), 2 pgs degraded) 2024-02-07T12:51:42.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:41 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:51:42.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:41 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:51:42.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:41 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:51:42.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:41 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:42.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:41 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:51:42.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:41 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:51:42.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:41 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:51:42.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:42 smithi019 ceph-mon[88137]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:42.844 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:51:42.844 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:42.844 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:42.844 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:42.844 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:42.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:42.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:51:42.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:42.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:51:42.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:42.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:51:42.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:42.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:42.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:42.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:42.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:42.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:42.845 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:42.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:42.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:42.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:51:42.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:51:42.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:51:42.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:51:42.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:51:42.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:51:42.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:51:42.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:51:42.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:51:42.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:51:42.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:51:42.846 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:51:42.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:51:42.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:51:42.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:51:42.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:51:42.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:51:42.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:42.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:51:42.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:42.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:42.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:42.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:42.847 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:42.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:51:42.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:42.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:51:42.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:42.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:51:42.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:42.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:42.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:42.848 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:42.848 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:42 smithi019 ceph-mon[82918]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:42.848 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:51:42.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:42.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:42.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:42.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:42.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:42.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:51:42.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:42.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:51:42.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:42.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:51:42.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:42.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:42.849 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:42.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:42.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:42.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:42.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:42.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:42.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:42.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:51:42.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:51:42.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:51:42.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:51:42.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:51:42.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:51:42.850 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:51:42.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:51:42.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:51:42.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:51:42.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:51:42.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:51:42.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:51:42.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:51:42.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:51:42.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:51:42.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:51:42.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:42.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:51:42.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:42.851 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:42.852 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:42.852 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:42.852 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:42.852 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:51:42.852 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:42.852 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:51:42.852 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:42.852 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:51:42.852 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:42.852 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:42.852 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:42.852 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:42.853 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:42.853 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:42.853 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:42.853 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:42.853 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:42.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:42 smithi019 ceph-mon[88137]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-02-07T12:51:42.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:42 smithi019 ceph-mon[88137]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-02-07T12:51:42.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:42 smithi019 ceph-mon[88137]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:51:42.853 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:42 smithi019 ceph-mon[88137]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:51:42.853 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:42.853 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:42.854 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:42.854 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:42.854 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:42.854 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:42 smithi019 ceph-mon[82918]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-02-07T12:51:42.854 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:42 smithi019 ceph-mon[82918]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-02-07T12:51:42.854 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:42 smithi019 ceph-mon[82918]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:51:42.854 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:42 smithi019 ceph-mon[82918]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:51:43.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:42 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:43.119 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:51:43.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:43.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:43.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:43.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:43.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:43.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:51:43.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:43.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:51:43.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:43.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:51:43.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:43.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:43.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:43.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:43.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:43.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:43.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:43.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:43.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:43.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:51:43.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:51:43.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:51:43.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:51:43.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:51:43.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:51:43.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:51:43.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:51:43.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:51:43.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:51:43.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:51:43.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:51:43.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:51:43.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:51:43.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:51:43.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:51:43.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:51:43.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:43.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:51:43.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:43.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:43.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:43.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:43.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:43.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:51:43.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:43.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:51:43.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:43.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:51:43.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:43.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:43.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:43.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:43.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:43.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:43.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:43.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:43.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:43.123 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:42 smithi172 ceph-mon[67636]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-02-07T12:51:43.123 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:42 smithi172 ceph-mon[67636]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-02-07T12:51:43.124 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:42 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:51:43.124 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:42 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:51:43.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:51:43.465+0000 7fb71d671700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:43.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:51:43.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:43.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:43.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:43.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:43.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:43.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:51:43.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:43.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:51:43.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:43.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:51:43.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:43.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:43.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:43.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:43.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:43.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:43.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:43.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:43.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:43.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:51:43.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:51:43.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:51:43.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:51:43.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:51:43.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:51:43.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:51:43.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:51:43.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:51:43.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:51:43.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:51:43.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:51:43.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:51:43.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:51:43.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:51:43.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:51:43.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:51:43.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:43.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:51:43.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:43.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:43.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:43.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:43.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:43.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:51:43.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:43.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:51:43.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:43.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:51:43.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:43.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:43.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:43.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:43.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:43.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:43.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:43.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:43.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:44.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:51:43] "GET /metrics HTTP/1.1" 200 38046 "" "Prometheus/2.43.0" 2024-02-07T12:51:45.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:44 smithi019 ceph-mon[82918]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:45.094 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:51:45.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:45.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:45.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:45.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:45.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:45.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:51:45.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:45.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:51:45.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:45.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:51:45.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:45.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:45.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:45.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:45.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:45.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:45.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:45.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:45.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:45.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:51:45.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:51:45.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:51:45.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:51:45.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:51:45.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:51:45.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:51:45.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:51:45.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:51:45.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:51:45.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:51:45.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:51:45.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:51:45.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:51:45.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:51:45.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:51:45.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:51:45.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:45.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:51:45.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:45.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:45.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:45.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:45.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:45.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:51:45.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:45.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:51:45.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:45.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:51:45.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:45.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:45.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:45.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:45.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:45.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:45.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:45.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:45.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:45.099 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:44 smithi019 ceph-mon[82918]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 878 B/s rd, 0 op/s 2024-02-07T12:51:45.099 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:44 smithi019 ceph-mon[88137]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:45.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:51:45.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:45.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:45.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:45.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:45.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:45.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:51:45.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:45.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:51:45.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:45.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:51:45.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:45.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:45.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:45.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:45.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:45.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:45.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:45.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:45.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:45.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:51:45.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:51:45.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:51:45.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:51:45.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:51:45.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:51:45.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:51:45.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:51:45.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:51:45.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:51:45.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:51:45.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:51:45.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:51:45.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:51:45.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:51:45.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:51:45.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:51:45.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:45.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:51:45.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:45.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:45.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:45.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:45.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:45.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:51:45.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:45.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:51:45.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:45.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:51:45.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:45.103 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:45.103 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:45.103 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:45.103 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:45.103 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:45.103 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:45.103 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:45.103 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:45.103 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:44 smithi019 ceph-mon[88137]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 878 B/s rd, 0 op/s 2024-02-07T12:51:45.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:44 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:45.119 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:51:45.119 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:45.119 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:45.119 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:45.119 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:45.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:45.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:51:45.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:45.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:51:45.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:45.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:51:45.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:45.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:45.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:45.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:45.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:45.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:45.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:45.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:45.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:45.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:51:45.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:51:45.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:51:45.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:51:45.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:51:45.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:51:45.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:51:45.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:51:45.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:51:45.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:51:45.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:51:45.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:51:45.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:51:45.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:51:45.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:51:45.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:51:45.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:51:45.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:51:45.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:51:45.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:51:45.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:51:45.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:51:45.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:45.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:45.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:51:45.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:51:45.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:51:45.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:51:45.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:51:45.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:51:45.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:51:45.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:51:45.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:51:45.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:51:45.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:51:45.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:51:45.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:51:45.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:51:45.123 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:44 smithi172 ceph-mon[67636]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 878 B/s rd, 0 op/s 2024-02-07T12:51:46.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:46 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:46.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:46 smithi019 ceph-mon[82918]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 878 B/s rd, 0 op/s 2024-02-07T12:51:46.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:46 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:46.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:46 smithi019 ceph-mon[88137]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 878 B/s rd, 0 op/s 2024-02-07T12:51:46.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:46 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:46.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:46 smithi172 ceph-mon[67636]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 878 B/s rd, 0 op/s 2024-02-07T12:51:47.520 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:47 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:51:47.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:47 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:51:47.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:47 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:51:48.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:48 smithi019 ceph-mon[82918]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-02-07T12:51:48.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:48 smithi019 ceph-mon[82918]: from='client.44394 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:48.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:48 smithi019 ceph-mon[88137]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-02-07T12:51:48.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:48 smithi019 ceph-mon[88137]: from='client.44394 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:48.462 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch ps' 2024-02-07T12:51:48.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:48 smithi172 ceph-mon[67636]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-02-07T12:51:48.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:48 smithi172 ceph-mon[67636]: from='client.44394 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:50.423 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:51:50.423 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (6m) 17s ago 27m 27.3M - 0.25.0 c8568f914cd2 fec9255f35e7 2024-02-07T12:51:50.423 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (5m) 3m ago 27m 82.9M - 9.4.12 a0ea5e3c46d9 aaffac63bc28 2024-02-07T12:51:50.423 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 unknown 17s ago 27m 63.4M - 3.5 e1d6a67b021e 5113aa77e3f2 2024-02-07T12:51:50.423 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283,8765 running (7m) 3m ago 32m 447M - 19.0.0-1252-g44458cce 79957b372307 5aa0c04fdfd7 2024-02-07T12:51:50.423 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (17m) 17s ago 34m 507M - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:51:50.424 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (3m) 17s ago 34m 51.7M 2048M 19.0.0-1252-g44458cce 79957b372307 8b1b0e21b1c5 2024-02-07T12:51:50.424 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (4m) 3m ago 32m 37.0M 2048M 19.0.0-1252-g44458cce 79957b372307 00f454bbac01 2024-02-07T12:51:50.424 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (3m) 17s ago 32m 42.2M 2048M 19.0.0-1252-g44458cce 79957b372307 6214003f6e08 2024-02-07T12:51:50.424 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (6m) 17s ago 28m 15.9M - 1.5.0 0da6a335fe13 c4bdcb1c4d49 2024-02-07T12:51:50.424 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (6m) 3m ago 28m 16.3M - 1.5.0 0da6a335fe13 04ea81f79d85 2024-02-07T12:51:50.424 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (108s) 17s ago 32m 54.4M 2993M 19.0.0-1252-g44458cce 79957b372307 78eddd34a903 2024-02-07T12:51:50.424 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (86s) 17s ago 31m 43.5M 2993M 19.0.0-1252-g44458cce 79957b372307 c37447e76c38 2024-02-07T12:51:50.424 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (22s) 17s ago 31m 12.3M 2993M 19.0.0-1252-g44458cce 79957b372307 90352897da85 2024-02-07T12:51:50.424 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (30m) 17s ago 30m 57.3M 2993M 17.2.0 e1d6a67b021e 0b6b674188ed 2024-02-07T12:51:50.424 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (30m) 3m ago 30m 55.7M 3248M 17.2.0 e1d6a67b021e df72cfd69b3b 2024-02-07T12:51:50.424 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (30m) 3m ago 30m 52.4M 3248M 17.2.0 e1d6a67b021e 3bdb22c07537 2024-02-07T12:51:50.424 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (29m) 3m ago 29m 52.1M 3248M 17.2.0 e1d6a67b021e 7859b651e0d9 2024-02-07T12:51:50.424 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (29m) 3m ago 29m 55.5M 3248M 17.2.0 e1d6a67b021e b301e7c75d73 2024-02-07T12:51:50.425 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (6m) 3m ago 28m 61.0M - 2.43.0 a07b618ecd1d c9b607eea1be 2024-02-07T12:51:50.425 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (27m) 17s ago 27m 79.8M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:51:50.425 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (27m) 3m ago 27m 78.3M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:51:51.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:50 smithi019 ceph-mon[82918]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:51:51.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:50 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:51.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:50 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:51:51.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:50 smithi019 ceph-mon[88137]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:51:51.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:50 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:51.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:50 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:51:51.144 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-02-07T12:51:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:50 smithi172 ceph-mon[67636]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:51:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:50 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:51:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:50 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:51:52.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:51 smithi019 ceph-mon[82918]: from='client.44400 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:52.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:51 smithi019 ceph-mon[88137]: from='client.44400 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:52.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:51 smithi172 ceph-mon[67636]: from='client.44400 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:53.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:52 smithi019 ceph-mon[88137]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:51:53.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:52 smithi019 ceph-mon[82918]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:51:53.155 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:51:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:52 smithi172 ceph-mon[67636]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:51:53.916 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 8'"'"'' 2024-02-07T12:51:54.232 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:53 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/3286862729' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:54.232 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:53 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/3286862729' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:54.232 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:51:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:51:53] "GET /metrics HTTP/1.1" 200 38183 "" "Prometheus/2.43.0" 2024-02-07T12:51:54.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:53 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/3286862729' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:51:55.149 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:54 smithi019 ceph-mon[88137]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:51:55.150 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:54 smithi019 ceph-mon[82918]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:51:55.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:54 smithi172 ceph-mon[67636]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:51:57.041 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:56 smithi019 ceph-mon[88137]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:51:57.041 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:56 smithi019 ceph-mon[88137]: from='client.44412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:57.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:56 smithi019 ceph-mon[82918]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:51:57.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:56 smithi019 ceph-mon[82918]: from='client.44412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:57.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:56 smithi172 ceph-mon[67636]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:51:57.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:56 smithi172 ceph-mon[67636]: from='client.44412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:51:57.989 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:51:58.332 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:57 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:51:58.332 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:57 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:51:58.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:57 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:51:58.660 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch upgrade status' 2024-02-07T12:51:59.236 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:51:58 smithi019 ceph-mon[88137]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:51:59.236 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:51:58 smithi019 ceph-mon[82918]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:51:59.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:51:58 smithi172 ceph-mon[67636]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:52:00.777 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:52:00.777 INFO:teuthology.orchestra.run.smithi019.stdout: "target_image": null, 2024-02-07T12:52:00.777 INFO:teuthology.orchestra.run.smithi019.stdout: "in_progress": false, 2024-02-07T12:52:00.777 INFO:teuthology.orchestra.run.smithi019.stdout: "which": "", 2024-02-07T12:52:00.777 INFO:teuthology.orchestra.run.smithi019.stdout: "services_complete": [], 2024-02-07T12:52:00.778 INFO:teuthology.orchestra.run.smithi019.stdout: "progress": null, 2024-02-07T12:52:00.778 INFO:teuthology.orchestra.run.smithi019.stdout: "message": "", 2024-02-07T12:52:00.778 INFO:teuthology.orchestra.run.smithi019.stdout: "is_paused": false 2024-02-07T12:52:00.778 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:52:01.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:00 smithi019 ceph-mon[88137]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:52:01.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:00 smithi019 ceph-mon[82918]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:52:01.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:00 smithi172 ceph-mon[67636]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:52:01.413 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph health detail' 2024-02-07T12:52:02.017 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:01 smithi019 ceph-mon[88137]: from='client.54355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:02.017 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:01 smithi019 ceph-mon[82918]: from='client.54355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:02.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:01 smithi172 ceph-mon[67636]: from='client.54355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:03.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:02 smithi019 ceph-mon[82918]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:52:03.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:02 smithi019 ceph-mon[88137]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:52:03.345 INFO:teuthology.orchestra.run.smithi019.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-02-07T12:52:03.345 INFO:teuthology.orchestra.run.smithi019.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-02-07T12:52:03.345 INFO:teuthology.orchestra.run.smithi019.stdout: daemon iscsi.foo.smithi019.svjimj on smithi019 is in unknown state 2024-02-07T12:52:03.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:02 smithi172 ceph-mon[67636]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:52:03.943 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd' 2024-02-07T12:52:04.246 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:03 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/1857764456' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:52:04.246 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:52:03] "GET /metrics HTTP/1.1" 200 38183 "" "Prometheus/2.43.0" 2024-02-07T12:52:04.246 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:03 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/1857764456' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:52:04.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:03 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/1857764456' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:52:05.184 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:04 smithi019 ceph-mon[88137]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:52:05.185 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:04 smithi019 ceph-mon[82918]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:52:05.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:04 smithi172 ceph-mon[67636]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:52:06.180 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:52:06.180 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:52:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:52:07.151 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:06 smithi019 ceph-mon[88137]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:52:07.151 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:06 smithi019 ceph-mon[88137]: from='client.44430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:07.151 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:06 smithi019 ceph-mon[82918]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:52:07.151 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:06 smithi019 ceph-mon[82918]: from='client.44430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:06 smithi172 ceph-mon[67636]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:52:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:06 smithi172 ceph-mon[67636]: from='client.44430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:08.000 INFO:teuthology.orchestra.run.smithi019.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:52:08.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:07 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:52:08.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:07 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:52:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:07 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:52:08.836 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-02-07T12:52:09.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:08 smithi019 ceph-mon[82918]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:52:09.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:08 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:09.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:08 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:52:09.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:08 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:52:09.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:08 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:52:09.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:08 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:09.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:08 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:52:09.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:08 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:52:09.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:08 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:52:09.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:08 smithi019 ceph-mon[88137]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:52:09.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:08 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:09.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:08 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:52:09.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:08 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:52:09.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:08 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:52:09.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:08 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:09.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:08 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:52:09.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:08 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:52:09.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:08 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:52:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:08 smithi172 ceph-mon[67636]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:52:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:08 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:08 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:52:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:08 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:52:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:08 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:52:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:08 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:08 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:52:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:08 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:52:09.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:08 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:52:10.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:09 smithi019 ceph-mon[88137]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:52:10.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:09 smithi019 ceph-mon[88137]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 975 B/s rd, 0 op/s 2024-02-07T12:52:10.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:09 smithi019 ceph-mon[88137]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:52:10.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:09 smithi019 ceph-mon[88137]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:52:10.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:09 smithi019 ceph-mon[88137]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:52:10.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:09 smithi019 ceph-mon[82918]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:52:10.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:09 smithi019 ceph-mon[82918]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 975 B/s rd, 0 op/s 2024-02-07T12:52:10.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:09 smithi019 ceph-mon[82918]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:52:10.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:09 smithi019 ceph-mon[82918]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:52:10.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:09 smithi019 ceph-mon[82918]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:52:10.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:09 smithi172 ceph-mon[67636]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:52:10.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:09 smithi172 ceph-mon[67636]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 975 B/s rd, 0 op/s 2024-02-07T12:52:10.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:09 smithi172 ceph-mon[67636]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:52:10.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:09 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:52:10.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:09 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:52:11.253 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:52:11.065+0000 7fb71d671700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:52:11.253 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:52:11.253 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:52:11.253 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:52:11.253 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:52:11.253 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:11.253 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:11.253 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:52:11.253 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:52:11.253 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:52:11.253 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:52:11.254 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:52:11.254 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:52:11.254 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:52:11.254 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:52:11.254 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:52:11.254 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:52:11.254 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:52:11.254 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:52:11.254 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:11.255 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:11.255 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:52:11.255 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:52:11.255 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:52:11.255 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:52:11.255 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:52:11.255 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:52:11.255 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:52:11.255 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:52:11.255 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:52:11.255 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:52:11.255 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:52:11.255 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:52:11.255 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:52:11.255 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:52:11.255 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:52:11.255 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:52:11.256 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:52:11.256 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:52:11.256 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:52:11.256 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:52:11.256 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:52:11.256 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:52:11.256 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:11.256 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:11.256 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:52:11.256 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:52:11.256 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:52:11.256 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:52:11.256 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:52:11.256 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:52:11.256 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:52:11.256 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:52:11.256 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:52:11.257 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:52:11.257 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:52:11.257 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:52:11.257 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:11.257 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:11.728 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:52:12.197 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-mon[82918]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 602 B/s rd, 0 op/s 2024-02-07T12:52:12.198 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:11 smithi019 ceph-mon[88137]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 602 B/s rd, 0 op/s 2024-02-07T12:52:12.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:11 smithi172 ceph-mon[67636]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 602 B/s rd, 0 op/s 2024-02-07T12:52:12.418 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:52:12.418 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (6m) 39s ago 28m 27.3M - 0.25.0 c8568f914cd2 fec9255f35e7 2024-02-07T12:52:12.418 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (6m) 4m ago 28m 82.9M - 9.4.12 a0ea5e3c46d9 aaffac63bc28 2024-02-07T12:52:12.418 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 unknown 39s ago 27m 63.4M - 3.5 e1d6a67b021e 5113aa77e3f2 2024-02-07T12:52:12.418 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283,8765 running (7m) 4m ago 32m 447M - 19.0.0-1252-g44458cce 79957b372307 5aa0c04fdfd7 2024-02-07T12:52:12.418 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (17m) 39s ago 34m 507M - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:52:12.418 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (4m) 39s ago 34m 51.7M 2048M 19.0.0-1252-g44458cce 79957b372307 8b1b0e21b1c5 2024-02-07T12:52:12.418 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (5m) 4m ago 33m 37.0M 2048M 19.0.0-1252-g44458cce 79957b372307 00f454bbac01 2024-02-07T12:52:12.418 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (3m) 39s ago 33m 42.2M 2048M 19.0.0-1252-g44458cce 79957b372307 6214003f6e08 2024-02-07T12:52:12.418 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (7m) 39s ago 29m 15.9M - 1.5.0 0da6a335fe13 c4bdcb1c4d49 2024-02-07T12:52:12.418 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (7m) 4m ago 28m 16.3M - 1.5.0 0da6a335fe13 04ea81f79d85 2024-02-07T12:52:12.418 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (2m) 39s ago 32m 54.4M 2993M 19.0.0-1252-g44458cce 79957b372307 78eddd34a903 2024-02-07T12:52:12.418 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (108s) 39s ago 32m 43.5M 2993M 19.0.0-1252-g44458cce 79957b372307 c37447e76c38 2024-02-07T12:52:12.418 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (44s) 39s ago 31m 12.3M 2993M 19.0.0-1252-g44458cce 79957b372307 90352897da85 2024-02-07T12:52:12.419 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (31m) 39s ago 31m 57.3M 2993M 17.2.0 e1d6a67b021e 0b6b674188ed 2024-02-07T12:52:12.419 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (30m) 4m ago 30m 55.7M 3248M 17.2.0 e1d6a67b021e df72cfd69b3b 2024-02-07T12:52:12.419 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (30m) 4m ago 30m 52.4M 3248M 17.2.0 e1d6a67b021e 3bdb22c07537 2024-02-07T12:52:12.419 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (30m) 4m ago 30m 52.1M 3248M 17.2.0 e1d6a67b021e 7859b651e0d9 2024-02-07T12:52:12.419 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (29m) 4m ago 29m 55.5M 3248M 17.2.0 e1d6a67b021e b301e7c75d73 2024-02-07T12:52:12.419 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (6m) 4m ago 29m 61.0M - 2.43.0 a07b618ecd1d c9b607eea1be 2024-02-07T12:52:12.419 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (27m) 39s ago 27m 79.8M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:52:12.419 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (27m) 4m ago 27m 78.3M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:52:12.835 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:52:12.835 INFO:teuthology.orchestra.run.smithi019.stdout: "mon": { 2024-02-07T12:52:12.836 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 3 2024-02-07T12:52:12.836 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:52:12.836 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr": { 2024-02-07T12:52:12.836 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:52:12.836 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:52:12.836 INFO:teuthology.orchestra.run.smithi019.stdout: "osd": { 2024-02-07T12:52:12.836 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-02-07T12:52:12.836 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 3 2024-02-07T12:52:12.836 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:52:12.836 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw": { 2024-02-07T12:52:12.836 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:52:12.836 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:52:12.836 INFO:teuthology.orchestra.run.smithi019.stdout: "overall": { 2024-02-07T12:52:12.836 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-02-07T12:52:12.837 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 8 2024-02-07T12:52:12.837 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:52:12.837 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:52:13.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:12 smithi019 ceph-mon[82918]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:52:13.094 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:52:13.094 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:52:13.094 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:52:13.094 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:52:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:52:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:52:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:52:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:52:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:52:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:52:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:52:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:52:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:52:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:52:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:52:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:52:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:13.095 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:13.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:52:13.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:52:13.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:52:13.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:52:13.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:52:13.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:52:13.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:52:13.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:52:13.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:52:13.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:52:13.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:52:13.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:52:13.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:52:13.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:52:13.096 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:52:13.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:52:13.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:52:13.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:52:13.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:52:13.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:52:13.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:52:13.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:52:13.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:13.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:13.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:52:13.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:52:13.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:52:13.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:52:13.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:52:13.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:52:13.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:52:13.097 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:52:13.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:52:13.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:52:13.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:52:13.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:52:13.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:13.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:13.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:12 smithi019 ceph-mon[82918]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:52:13.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:12 smithi019 ceph-mon[82918]: from='client.54373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:13.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:12 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/1003669119' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:13.098 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:12 smithi019 ceph-mon[88137]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:52:13.098 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:52:13.098 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:52:13.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:52:13.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:52:13.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:13.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:13.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:52:13.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:52:13.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:52:13.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:52:13.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:52:13.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:52:13.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:52:13.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:52:13.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:52:13.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:52:13.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:52:13.099 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:52:13.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:13.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:13.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:52:13.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:52:13.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:52:13.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:52:13.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:52:13.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:52:13.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:52:13.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:52:13.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:52:13.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:52:13.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:52:13.100 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:52:13.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:52:13.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:52:13.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:52:13.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:52:13.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:52:13.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:52:13.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:52:13.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:52:13.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:52:13.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:52:13.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:13.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:13.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:52:13.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:52:13.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:52:13.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:52:13.101 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:52:13.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:52:13.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:52:13.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:52:13.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:52:13.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:52:13.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:52:13.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:52:13.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:13.102 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:13.102 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:12 smithi019 ceph-mon[88137]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:52:13.102 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:12 smithi019 ceph-mon[88137]: from='client.54373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:13.102 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:12 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/1003669119' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:13.196 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:52:13.197 INFO:teuthology.orchestra.run.smithi019.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:52:13.197 INFO:teuthology.orchestra.run.smithi019.stdout: "in_progress": true, 2024-02-07T12:52:13.197 INFO:teuthology.orchestra.run.smithi019.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-02-07T12:52:13.197 INFO:teuthology.orchestra.run.smithi019.stdout: "services_complete": [], 2024-02-07T12:52:13.197 INFO:teuthology.orchestra.run.smithi019.stdout: "progress": "3/8 daemons upgraded", 2024-02-07T12:52:13.197 INFO:teuthology.orchestra.run.smithi019.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 image", 2024-02-07T12:52:13.197 INFO:teuthology.orchestra.run.smithi019.stdout: "is_paused": false 2024-02-07T12:52:13.197 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:52:13.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:12 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:52:13.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:52:13.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:52:13.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:52:13.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:52:13.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:13.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:13.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:52:13.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:52:13.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:52:13.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:52:13.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:52:13.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:52:13.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:52:13.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:52:13.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:52:13.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:52:13.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:52:13.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:52:13.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:13.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:13.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:52:13.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:52:13.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:52:13.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:52:13.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:52:13.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:52:13.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:52:13.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:52:13.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:52:13.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:52:13.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:52:13.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:52:13.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:52:13.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:52:13.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:52:13.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:52:13.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:52:13.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:52:13.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:52:13.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:52:13.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:52:13.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:52:13.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:13.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:13.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:52:13.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:52:13.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:52:13.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:52:13.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:52:13.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:52:13.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:52:13.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:52:13.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:52:13.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:52:13.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:52:13.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:52:13.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:13.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:13.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:12 smithi172 ceph-mon[67636]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:52:13.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:12 smithi172 ceph-mon[67636]: from='client.54373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:13.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:12 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/1003669119' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:14.060 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:52:13] "GET /metrics HTTP/1.1" 200 38181 "" "Prometheus/2.43.0" 2024-02-07T12:52:14.060 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:13 smithi019 ceph-mon[82918]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:52:14.060 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:13 smithi019 ceph-mon[82918]: from='client.44442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:14.060 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:13 smithi019 ceph-mon[82918]: from='client.54385 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:14.060 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:13 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:14.060 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:13 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:52:14.061 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:13 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:14.061 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:13 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:14.061 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:13 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:14.061 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:13 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:14.061 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:13 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:14.061 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:13 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:14.061 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:13 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-02-07T12:52:14.061 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:13 smithi019 ceph-mon[88137]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:52:14.061 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:13 smithi019 ceph-mon[88137]: from='client.44442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:14.061 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:13 smithi019 ceph-mon[88137]: from='client.54385 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:14.061 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:13 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:14.061 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:13 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:52:14.061 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:13 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:14.061 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:13 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:14.062 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:13 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:14.062 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:13 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:14.062 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:13 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:14.062 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:13 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:14.062 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:13 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-02-07T12:52:14.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:13 smithi172 ceph-mon[67636]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:52:14.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:13 smithi172 ceph-mon[67636]: from='client.44442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:14.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:13 smithi172 ceph-mon[67636]: from='client.54385 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:14.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:13 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:14.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:13 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:52:14.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:13 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:14.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:13 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:14.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:13 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:14.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:13 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:14.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:13 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:14.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:13 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:14.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:13 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-02-07T12:52:15.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:14 smithi019 ceph-mon[88137]: Upgrade: Target is version 19.0.0-1252-g44458cce (squid) 2024-02-07T12:52:15.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:14 smithi019 ceph-mon[88137]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe'] 2024-02-07T12:52:15.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:14 smithi019 ceph-mon[88137]: from='client.44460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:15.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:14 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all mgr 2024-02-07T12:52:15.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:14 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all mon 2024-02-07T12:52:15.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:14 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all crash 2024-02-07T12:52:15.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:14 smithi019 ceph-mon[88137]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-02-07T12:52:15.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:14 smithi019 ceph-mon[88137]: Upgrade: osd.3 is safe to restart 2024-02-07T12:52:15.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:14 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:15.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:14 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-02-07T12:52:15.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:14 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:52:15.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:14 smithi019 ceph-mon[82918]: Upgrade: Target is version 19.0.0-1252-g44458cce (squid) 2024-02-07T12:52:15.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:14 smithi019 ceph-mon[82918]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe'] 2024-02-07T12:52:15.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:14 smithi019 ceph-mon[82918]: from='client.44460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:15.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:14 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all mgr 2024-02-07T12:52:15.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:14 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all mon 2024-02-07T12:52:15.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:14 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all crash 2024-02-07T12:52:15.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:14 smithi019 ceph-mon[82918]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-02-07T12:52:15.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:14 smithi019 ceph-mon[82918]: Upgrade: osd.3 is safe to restart 2024-02-07T12:52:15.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:14 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:15.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:14 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-02-07T12:52:15.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:14 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:52:15.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:14 smithi172 ceph-mon[67636]: Upgrade: Target is version 19.0.0-1252-g44458cce (squid) 2024-02-07T12:52:15.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:14 smithi172 ceph-mon[67636]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe'] 2024-02-07T12:52:15.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:14 smithi172 ceph-mon[67636]: from='client.44460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:15.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:14 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all mgr 2024-02-07T12:52:15.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:14 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all mon 2024-02-07T12:52:15.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:14 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all crash 2024-02-07T12:52:15.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:14 smithi172 ceph-mon[67636]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-02-07T12:52:15.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:14 smithi172 ceph-mon[67636]: Upgrade: osd.3 is safe to restart 2024-02-07T12:52:15.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:14 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:15.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:14 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-02-07T12:52:15.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:14 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:52:16.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:15 smithi019 ceph-mon[82918]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:52:16.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:15 smithi019 ceph-mon[82918]: Upgrade: Updating osd.3 2024-02-07T12:52:16.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:15 smithi019 ceph-mon[82918]: Deploying daemon osd.3 on smithi019 2024-02-07T12:52:16.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:15 smithi019 ceph-mon[88137]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:52:16.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:15 smithi019 ceph-mon[88137]: Upgrade: Updating osd.3 2024-02-07T12:52:16.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:15 smithi019 ceph-mon[88137]: Deploying daemon osd.3 on smithi019 2024-02-07T12:52:16.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:15 smithi172 ceph-mon[67636]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:52:16.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:15 smithi172 ceph-mon[67636]: Upgrade: Updating osd.3 2024-02-07T12:52:16.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:15 smithi172 ceph-mon[67636]: Deploying daemon osd.3 on smithi019 2024-02-07T12:52:18.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:17 smithi019 ceph-mon[88137]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:52:18.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:17 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:52:18.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:17 smithi019 ceph-mon[82918]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:52:18.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:17 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:52:18.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:17 smithi172 ceph-mon[67636]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:52:18.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:17 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:52:18.424 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:18 smithi019 systemd[1]: Stopping Ceph osd.3 for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:52:18.844 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3[40980]: 2024-02-07T12:52:18.421+0000 7ffa6263c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:52:18.844 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3[40980]: 2024-02-07T12:52:18.421+0000 7ffa6263c700 -1 osd.3 120 *** Got signal Terminated *** 2024-02-07T12:52:18.844 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3[40980]: 2024-02-07T12:52:18.421+0000 7ffa6263c700 -1 osd.3 120 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:52:19.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:18 smithi019 ceph-mon[82918]: osd.3 marked itself down and dead 2024-02-07T12:52:19.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:18 smithi019 ceph-mon[88137]: osd.3 marked itself down and dead 2024-02-07T12:52:19.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:18 smithi172 ceph-mon[67636]: osd.3 marked itself down and dead 2024-02-07T12:52:19.758 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:19 smithi019 podman[114474]: 2024-02-07 12:52:19.495952958 +0000 UTC m=+1.182514695 container died 0b6b674188edcba75dbcf1c7bef5713c89e9c6f5e6c9f2cf1a36d8e5a5069dad (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3, name=centos-stream, ceph=True, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., vcs-type=git, io.buildah.version=1.19.8, RELEASE=HEAD, release=754, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, io.openshift.tags=base centos centos-stream, architecture=x86_64) 2024-02-07T12:52:20.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:19 smithi019 ceph-mon[88137]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:52:20.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:19 smithi019 ceph-mon[88137]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:52:20.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:19 smithi019 ceph-mon[88137]: failed to encode map e121 with expected crc 2024-02-07T12:52:20.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:19 smithi019 ceph-mon[88137]: failed to encode map e121 with expected crc 2024-02-07T12:52:20.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:19 smithi019 ceph-mon[88137]: osdmap e121: 8 total, 7 up, 8 in 2024-02-07T12:52:20.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:19 smithi019 ceph-mon[88137]: failed to encode map e121 with expected crc 2024-02-07T12:52:20.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:19 smithi019 ceph-mon[88137]: failed to encode map e121 with expected crc 2024-02-07T12:52:20.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:19 smithi019 ceph-mon[88137]: failed to encode map e121 with expected crc 2024-02-07T12:52:20.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:19 smithi019 ceph-mon[88137]: failed to encode map e121 with expected crc 2024-02-07T12:52:20.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:19 smithi019 ceph-mon[88137]: failed to encode map e121 with expected crc 2024-02-07T12:52:20.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:19 smithi019 ceph-mon[82918]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:52:20.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:19 smithi019 ceph-mon[82918]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:52:20.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:19 smithi019 ceph-mon[82918]: failed to encode map e121 with expected crc 2024-02-07T12:52:20.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:19 smithi019 ceph-mon[82918]: failed to encode map e121 with expected crc 2024-02-07T12:52:20.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:19 smithi019 ceph-mon[82918]: osdmap e121: 8 total, 7 up, 8 in 2024-02-07T12:52:20.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:19 smithi019 ceph-mon[82918]: failed to encode map e121 with expected crc 2024-02-07T12:52:20.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:19 smithi019 ceph-mon[82918]: failed to encode map e121 with expected crc 2024-02-07T12:52:20.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:19 smithi019 ceph-mon[82918]: failed to encode map e121 with expected crc 2024-02-07T12:52:20.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:19 smithi019 ceph-mon[82918]: failed to encode map e121 with expected crc 2024-02-07T12:52:20.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:19 smithi019 ceph-mon[82918]: failed to encode map e121 with expected crc 2024-02-07T12:52:20.096 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:19 smithi019 podman[114474]: 2024-02-07 12:52:19.947434817 +0000 UTC m=+1.633996590 container cleanup 0b6b674188edcba75dbcf1c7bef5713c89e9c6f5e6c9f2cf1a36d8e5a5069dad (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, RELEASE=HEAD, distribution-scope=public, version=8, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, ceph=True, io.openshift.expose-services=, GIT_CLEAN=True) 2024-02-07T12:52:20.096 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:19 smithi019 bash[114474]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3 2024-02-07T12:52:20.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:19 smithi172 ceph-mon[67636]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:52:20.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:19 smithi172 ceph-mon[67636]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:52:20.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:19 smithi172 ceph-mon[67636]: failed to encode map e121 with expected crc 2024-02-07T12:52:20.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:19 smithi172 ceph-mon[67636]: failed to encode map e121 with expected crc 2024-02-07T12:52:20.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:19 smithi172 ceph-mon[67636]: osdmap e121: 8 total, 7 up, 8 in 2024-02-07T12:52:20.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:19 smithi172 ceph-mon[67636]: failed to encode map e121 with expected crc 2024-02-07T12:52:20.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:19 smithi172 ceph-mon[67636]: failed to encode map e121 with expected crc 2024-02-07T12:52:20.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:19 smithi172 ceph-mon[67636]: failed to encode map e121 with expected crc 2024-02-07T12:52:20.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:19 smithi172 ceph-mon[67636]: failed to encode map e121 with expected crc 2024-02-07T12:52:20.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:19 smithi172 ceph-mon[67636]: failed to encode map e121 with expected crc 2024-02-07T12:52:20.568 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:20 smithi019 podman[114491]: 2024-02-07 12:52:20.249122359 +0000 UTC m=+0.751969909 container remove 0b6b674188edcba75dbcf1c7bef5713c89e9c6f5e6c9f2cf1a36d8e5a5069dad (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3, com.redhat.component=centos-stream-container, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, io.openshift.tags=base centos centos-stream, version=8, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64) 2024-02-07T12:52:20.829 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:20 smithi019 podman[114581]: 2024-02-07 12:52:20.566787001 +0000 UTC m=+0.104286900 container create 5fa93358f8f5265cc3119f61aaca42d85cbd607f724cf1eda1a2a06c03d3145f (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, GIT_CLEAN=True, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-02-07T12:52:20.830 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:20 smithi019 podman[114581]: 2024-02-07 12:52:20.479794541 +0000 UTC m=+0.017294399 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:52:20.830 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:20 smithi019 podman[114581]: 2024-02-07 12:52:20.82827369 +0000 UTC m=+0.365773550 container init 5fa93358f8f5265cc3119f61aaca42d85cbd607f724cf1eda1a2a06c03d3145f (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:52:21.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:20 smithi019 ceph-mon[88137]: failed to encode map e122 with expected crc 2024-02-07T12:52:21.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:20 smithi019 ceph-mon[88137]: failed to encode map e122 with expected crc 2024-02-07T12:52:21.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:20 smithi019 ceph-mon[88137]: failed to encode map e122 with expected crc 2024-02-07T12:52:21.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:20 smithi019 ceph-mon[88137]: failed to encode map e122 with expected crc 2024-02-07T12:52:21.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:20 smithi019 ceph-mon[88137]: failed to encode map e122 with expected crc 2024-02-07T12:52:21.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:20 smithi019 ceph-mon[88137]: osdmap e122: 8 total, 7 up, 8 in 2024-02-07T12:52:21.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:20 smithi019 ceph-mon[88137]: failed to encode map e122 with expected crc 2024-02-07T12:52:21.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:20 smithi019 ceph-mon[88137]: failed to encode map e122 with expected crc 2024-02-07T12:52:21.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:20 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:21.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:20 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:52:21.095 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:20 smithi019 podman[114581]: 2024-02-07 12:52:20.831812599 +0000 UTC m=+0.369312449 container start 5fa93358f8f5265cc3119f61aaca42d85cbd607f724cf1eda1a2a06c03d3145f (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=) 2024-02-07T12:52:21.095 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:20 smithi019 podman[114581]: 2024-02-07 12:52:20.919394046 +0000 UTC m=+0.456893915 container attach 5fa93358f8f5265cc3119f61aaca42d85cbd607f724cf1eda1a2a06c03d3145f (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240131, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux ) 2024-02-07T12:52:21.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:20 smithi019 ceph-mon[82918]: failed to encode map e122 with expected crc 2024-02-07T12:52:21.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:20 smithi019 ceph-mon[82918]: failed to encode map e122 with expected crc 2024-02-07T12:52:21.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:20 smithi019 ceph-mon[82918]: failed to encode map e122 with expected crc 2024-02-07T12:52:21.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:20 smithi019 ceph-mon[82918]: failed to encode map e122 with expected crc 2024-02-07T12:52:21.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:20 smithi019 ceph-mon[82918]: failed to encode map e122 with expected crc 2024-02-07T12:52:21.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:20 smithi019 ceph-mon[82918]: osdmap e122: 8 total, 7 up, 8 in 2024-02-07T12:52:21.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:20 smithi019 ceph-mon[82918]: failed to encode map e122 with expected crc 2024-02-07T12:52:21.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:20 smithi019 ceph-mon[82918]: failed to encode map e122 with expected crc 2024-02-07T12:52:21.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:20 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:21.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:20 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:52:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:20 smithi172 ceph-mon[67636]: failed to encode map e122 with expected crc 2024-02-07T12:52:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:20 smithi172 ceph-mon[67636]: failed to encode map e122 with expected crc 2024-02-07T12:52:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:20 smithi172 ceph-mon[67636]: failed to encode map e122 with expected crc 2024-02-07T12:52:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:20 smithi172 ceph-mon[67636]: failed to encode map e122 with expected crc 2024-02-07T12:52:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:20 smithi172 ceph-mon[67636]: failed to encode map e122 with expected crc 2024-02-07T12:52:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:20 smithi172 ceph-mon[67636]: osdmap e122: 8 total, 7 up, 8 in 2024-02-07T12:52:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:20 smithi172 ceph-mon[67636]: failed to encode map e122 with expected crc 2024-02-07T12:52:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:20 smithi172 ceph-mon[67636]: failed to encode map e122 with expected crc 2024-02-07T12:52:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:20 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:20 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:52:21.424 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:21 smithi019 podman[114581]: 2024-02-07 12:52:21.141278227 +0000 UTC m=+0.678778083 container died 5fa93358f8f5265cc3119f61aaca42d85cbd607f724cf1eda1a2a06c03d3145f (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3-deactivate, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:52:21.987 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:21 smithi019 podman[114624]: 2024-02-07 12:52:21.707022747 +0000 UTC m=+0.561093155 container remove 5fa93358f8f5265cc3119f61aaca42d85cbd607f724cf1eda1a2a06c03d3145f (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131, ceph=True, org.label-schema.vendor=CentOS) 2024-02-07T12:52:21.987 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:21 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.3.service: Deactivated successfully. 2024-02-07T12:52:21.987 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:21 smithi019 systemd[1]: Stopped Ceph osd.3 for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:52:21.987 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:21 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.3.service: Consumed 17.492s CPU time. 2024-02-07T12:52:21.987 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:21 smithi019 systemd[1]: Starting Ceph osd.3 for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:52:22.252 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:21 smithi019 ceph-mon[82918]: pgmap v169: 161 pgs: 27 stale+active+clean, 134 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-02-07T12:52:22.252 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:21 smithi019 ceph-mon[88137]: pgmap v169: 161 pgs: 27 stale+active+clean, 134 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-02-07T12:52:22.253 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:22 smithi019 podman[114730]: 2024-02-07 12:52:22.141237245 +0000 UTC m=+0.105031002 container create 6396996a0bac951e4b0510f869ad56a8c4ee74291411b7c6b89d1d938ce90fb9 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3-activate, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:52:22.253 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:22 smithi019 podman[114730]: 2024-02-07 12:52:22.054751335 +0000 UTC m=+0.018545100 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:52:22.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:21 smithi172 ceph-mon[67636]: pgmap v169: 161 pgs: 27 stale+active+clean, 134 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-02-07T12:52:22.594 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:22 smithi019 podman[114730]: 2024-02-07 12:52:22.339367925 +0000 UTC m=+0.303161683 container init 6396996a0bac951e4b0510f869ad56a8c4ee74291411b7c6b89d1d938ce90fb9 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3-activate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.vendor=CentOS, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2) 2024-02-07T12:52:22.594 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:22 smithi019 podman[114730]: 2024-02-07 12:52:22.342718141 +0000 UTC m=+0.306511904 container start 6396996a0bac951e4b0510f869ad56a8c4ee74291411b7c6b89d1d938ce90fb9 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, GIT_CLEAN=True) 2024-02-07T12:52:22.594 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:22 smithi019 podman[114730]: 2024-02-07 12:52:22.427132493 +0000 UTC m=+0.390926281 container attach 6396996a0bac951e4b0510f869ad56a8c4ee74291411b7c6b89d1d938ce90fb9 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3-activate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:52:23.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:22 smithi019 ceph-mon[88137]: pgmap v170: 161 pgs: 19 active+undersized+degraded, 7 stale+active+clean, 27 active+undersized, 108 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 62/627 objects degraded (9.888%) 2024-02-07T12:52:23.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:22 smithi019 ceph-mon[82918]: pgmap v170: 161 pgs: 19 active+undersized+degraded, 7 stale+active+clean, 27 active+undersized, 108 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 62/627 objects degraded (9.888%) 2024-02-07T12:52:23.095 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:22 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3-activate[114744]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-02-07T12:52:23.095 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:22 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3-activate[114744]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-02-07T12:52:23.095 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:22 smithi019 bash[114730]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-02-07T12:52:23.095 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:22 smithi019 bash[114730]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-02-07T12:52:23.095 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:22 smithi019 bash[114730]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-02-07T12:52:23.095 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:22 smithi019 bash[114730]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-02-07T12:52:23.095 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:22 smithi019 bash[114730]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-02-07T12:52:23.095 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:22 smithi019 bash[114730]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-02-07T12:52:23.095 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:22 smithi019 bash[114730]: --> ceph-volume raw activate successful for osd ID: 3 2024-02-07T12:52:23.095 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:22 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3-activate[114744]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-02-07T12:52:23.096 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:22 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3-activate[114744]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-02-07T12:52:23.096 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:22 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3-activate[114744]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-02-07T12:52:23.096 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:22 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3-activate[114744]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-02-07T12:52:23.096 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:22 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3-activate[114744]: --> ceph-volume raw activate successful for osd ID: 3 2024-02-07T12:52:23.096 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:22 smithi019 podman[114730]: 2024-02-07 12:52:22.814202278 +0000 UTC m=+0.777996035 container died 6396996a0bac951e4b0510f869ad56a8c4ee74291411b7c6b89d1d938ce90fb9 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3-activate, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240131, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-02-07T12:52:23.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:22 smithi172 ceph-mon[67636]: pgmap v170: 161 pgs: 19 active+undersized+degraded, 7 stale+active+clean, 27 active+undersized, 108 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 62/627 objects degraded (9.888%) 2024-02-07T12:52:23.568 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:23 smithi019 podman[114868]: 2024-02-07 12:52:23.35742784 +0000 UTC m=+0.538186506 container remove 6396996a0bac951e4b0510f869ad56a8c4ee74291411b7c6b89d1d938ce90fb9 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3-activate, org.label-schema.build-date=20240131, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:52:23.844 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:23 smithi019 podman[114930]: 2024-02-07 12:52:23.566615395 +0000 UTC m=+0.111136192 container create 067cd6b3fd2f16613dfbb8ca6d758f04a49e7f52d2fc535c2a247db069de3bf9 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-02-07T12:52:23.844 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:23 smithi019 podman[114930]: 2024-02-07 12:52:23.474001933 +0000 UTC m=+0.018522733 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:52:23.844 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:23 smithi019 podman[114930]: 2024-02-07 12:52:23.785731857 +0000 UTC m=+0.330252654 container init 067cd6b3fd2f16613dfbb8ca6d758f04a49e7f52d2fc535c2a247db069de3bf9 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3, io.buildah.version=1.33.2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:52:23.844 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:23 smithi019 podman[114930]: 2024-02-07 12:52:23.788575169 +0000 UTC m=+0.333095977 container start 067cd6b3fd2f16613dfbb8ca6d758f04a49e7f52d2fc535c2a247db069de3bf9 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3, org.label-schema.build-date=20240131, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, org.label-schema.schema-version=1.0) 2024-02-07T12:52:23.845 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:23 smithi019 ceph-osd[114949]: -- 172.21.15.19:0/437594799 <== mon.1 v2:172.21.15.19:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5610277e76c0 con 0x5610277b8c00 2024-02-07T12:52:24.334 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:52:23] "GET /metrics HTTP/1.1" 200 38183 "" "Prometheus/2.43.0" 2024-02-07T12:52:24.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:23 smithi019 ceph-mon[88137]: Health check failed: Degraded data redundancy: 62/627 objects degraded (9.888%), 19 pgs degraded (PG_DEGRADED) 2024-02-07T12:52:24.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:23 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:24.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:23 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:24.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:23 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:52:24.335 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:23 smithi019 bash[114930]: 067cd6b3fd2f16613dfbb8ca6d758f04a49e7f52d2fc535c2a247db069de3bf9 2024-02-07T12:52:24.335 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:23 smithi019 systemd[1]: Started Ceph osd.3 for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:52:24.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:23 smithi019 ceph-mon[82918]: Health check failed: Degraded data redundancy: 62/627 objects degraded (9.888%), 19 pgs degraded (PG_DEGRADED) 2024-02-07T12:52:24.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:23 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:24.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:23 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:24.335 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:23 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:52:24.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:23 smithi172 ceph-mon[67636]: Health check failed: Degraded data redundancy: 62/627 objects degraded (9.888%), 19 pgs degraded (PG_DEGRADED) 2024-02-07T12:52:24.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:23 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:24.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:23 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:24.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:23 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:52:24.594 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:24 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3[114945]: 2024-02-07T12:52:24.331+0000 7fda6c5f27c0 -1 Falling back to public interface 2024-02-07T12:52:26.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:25 smithi019 ceph-mon[82918]: pgmap v171: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 82/627 objects degraded (13.078%) 2024-02-07T12:52:26.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:25 smithi019 ceph-mon[88137]: pgmap v171: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 82/627 objects degraded (13.078%) 2024-02-07T12:52:26.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:25 smithi172 ceph-mon[67636]: pgmap v171: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 82/627 objects degraded (13.078%) 2024-02-07T12:52:27.344 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:27 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3[114945]: 2024-02-07T12:52:27.061+0000 7fda6c5f27c0 -1 osd.3 0 read_superblock omap replica is missing. 2024-02-07T12:52:27.344 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:27 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3[114945]: 2024-02-07T12:52:27.094+0000 7fda6c5f27c0 -1 osd.3 120 log_to_monitors true 2024-02-07T12:52:28.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:27 smithi019 ceph-mon[82918]: pgmap v172: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 82/627 objects degraded (13.078%) 2024-02-07T12:52:28.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:27 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:52:28.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:27 smithi019 ceph-mon[82918]: from='osd.3 [v2:172.21.15.19:6826/3954572030,v1:172.21.15.19:6827/3954572030]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-02-07T12:52:28.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:27 smithi019 ceph-mon[82918]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-02-07T12:52:28.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:27 smithi019 ceph-mon[88137]: pgmap v172: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 82/627 objects degraded (13.078%) 2024-02-07T12:52:28.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:27 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:52:28.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:27 smithi019 ceph-mon[88137]: from='osd.3 [v2:172.21.15.19:6826/3954572030,v1:172.21.15.19:6827/3954572030]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-02-07T12:52:28.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:27 smithi019 ceph-mon[88137]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-02-07T12:52:28.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:27 smithi172 ceph-mon[67636]: pgmap v172: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 82/627 objects degraded (13.078%) 2024-02-07T12:52:28.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:27 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:52:28.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:27 smithi172 ceph-mon[67636]: from='osd.3 [v2:172.21.15.19:6826/3954572030,v1:172.21.15.19:6827/3954572030]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-02-07T12:52:28.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:27 smithi172 ceph-mon[67636]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-02-07T12:52:28.844 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:52:28 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3[114945]: 2024-02-07T12:52:28.399+0000 7fda59a49700 -1 osd.3 120 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:52:29.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:28 smithi019 ceph-mon[88137]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-02-07T12:52:29.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:28 smithi019 ceph-mon[88137]: osdmap e123: 8 total, 7 up, 8 in 2024-02-07T12:52:29.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:28 smithi019 ceph-mon[88137]: failed to encode map e123 with expected crc 2024-02-07T12:52:29.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:28 smithi019 ceph-mon[88137]: from='osd.3 [v2:172.21.15.19:6826/3954572030,v1:172.21.15.19:6827/3954572030]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:52:29.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:28 smithi019 ceph-mon[88137]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:52:29.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:28 smithi019 ceph-mon[88137]: failed to encode map e123 with expected crc 2024-02-07T12:52:29.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:28 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:29.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:28 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:29.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:28 smithi019 ceph-mon[88137]: from='osd.3 ' entity='osd.3' 2024-02-07T12:52:29.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:28 smithi019 ceph-mon[88137]: failed to encode map e123 with expected crc 2024-02-07T12:52:29.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:28 smithi019 ceph-mon[82918]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-02-07T12:52:29.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:28 smithi019 ceph-mon[82918]: osdmap e123: 8 total, 7 up, 8 in 2024-02-07T12:52:29.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:28 smithi019 ceph-mon[82918]: failed to encode map e123 with expected crc 2024-02-07T12:52:29.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:28 smithi019 ceph-mon[82918]: from='osd.3 [v2:172.21.15.19:6826/3954572030,v1:172.21.15.19:6827/3954572030]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:52:29.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:28 smithi019 ceph-mon[82918]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:52:29.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:28 smithi019 ceph-mon[82918]: failed to encode map e123 with expected crc 2024-02-07T12:52:29.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:28 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:29.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:28 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:29.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:28 smithi019 ceph-mon[82918]: from='osd.3 ' entity='osd.3' 2024-02-07T12:52:29.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:28 smithi019 ceph-mon[82918]: failed to encode map e123 with expected crc 2024-02-07T12:52:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:28 smithi172 ceph-mon[67636]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-02-07T12:52:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:28 smithi172 ceph-mon[67636]: osdmap e123: 8 total, 7 up, 8 in 2024-02-07T12:52:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:28 smithi172 ceph-mon[67636]: failed to encode map e123 with expected crc 2024-02-07T12:52:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:28 smithi172 ceph-mon[67636]: from='osd.3 [v2:172.21.15.19:6826/3954572030,v1:172.21.15.19:6827/3954572030]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:52:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:28 smithi172 ceph-mon[67636]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi019", "root=default"]}]: dispatch 2024-02-07T12:52:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:28 smithi172 ceph-mon[67636]: failed to encode map e123 with expected crc 2024-02-07T12:52:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:28 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:28 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:28 smithi172 ceph-mon[67636]: from='osd.3 ' entity='osd.3' 2024-02-07T12:52:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:28 smithi172 ceph-mon[67636]: failed to encode map e123 with expected crc 2024-02-07T12:52:30.165 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:29 smithi019 ceph-mon[88137]: pgmap v174: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 168 MiB used, 715 GiB / 715 GiB avail; 82/627 objects degraded (13.078%) 2024-02-07T12:52:30.165 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:29 smithi019 ceph-mon[88137]: failed to encode map e123 with expected crc 2024-02-07T12:52:30.165 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:29 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:30.165 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:29 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:30.165 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:29 smithi019 ceph-mon[88137]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:52:30.165 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:29 smithi019 ceph-mon[88137]: osd.3 [v2:172.21.15.19:6826/3954572030,v1:172.21.15.19:6827/3954572030] boot 2024-02-07T12:52:30.165 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:29 smithi019 ceph-mon[88137]: osdmap e124: 8 total, 8 up, 8 in 2024-02-07T12:52:30.165 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:29 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:52:30.165 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:29 smithi019 ceph-mon[88137]: Health check update: Degraded data redundancy: 82/627 objects degraded (13.078%), 26 pgs degraded (PG_DEGRADED) 2024-02-07T12:52:30.166 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:29 smithi019 ceph-mon[82918]: pgmap v174: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 168 MiB used, 715 GiB / 715 GiB avail; 82/627 objects degraded (13.078%) 2024-02-07T12:52:30.166 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:29 smithi019 ceph-mon[82918]: failed to encode map e123 with expected crc 2024-02-07T12:52:30.166 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:29 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:30.166 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:29 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:30.166 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:29 smithi019 ceph-mon[82918]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:52:30.166 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:29 smithi019 ceph-mon[82918]: osd.3 [v2:172.21.15.19:6826/3954572030,v1:172.21.15.19:6827/3954572030] boot 2024-02-07T12:52:30.166 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:29 smithi019 ceph-mon[82918]: osdmap e124: 8 total, 8 up, 8 in 2024-02-07T12:52:30.166 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:29 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:52:30.166 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:29 smithi019 ceph-mon[82918]: Health check update: Degraded data redundancy: 82/627 objects degraded (13.078%), 26 pgs degraded (PG_DEGRADED) 2024-02-07T12:52:30.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:29 smithi172 ceph-mon[67636]: pgmap v174: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 168 MiB used, 715 GiB / 715 GiB avail; 82/627 objects degraded (13.078%) 2024-02-07T12:52:30.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:29 smithi172 ceph-mon[67636]: failed to encode map e123 with expected crc 2024-02-07T12:52:30.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:29 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:30.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:29 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:30.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:29 smithi172 ceph-mon[67636]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:52:30.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:29 smithi172 ceph-mon[67636]: osd.3 [v2:172.21.15.19:6826/3954572030,v1:172.21.15.19:6827/3954572030] boot 2024-02-07T12:52:30.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:29 smithi172 ceph-mon[67636]: osdmap e124: 8 total, 8 up, 8 in 2024-02-07T12:52:30.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:29 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:52:30.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:29 smithi172 ceph-mon[67636]: Health check update: Degraded data redundancy: 82/627 objects degraded (13.078%), 26 pgs degraded (PG_DEGRADED) 2024-02-07T12:52:31.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:30 smithi019 ceph-mon[88137]: failed to encode map e124 with expected crc 2024-02-07T12:52:31.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:30 smithi019 ceph-mon[88137]: failed to encode map e124 with expected crc 2024-02-07T12:52:31.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:30 smithi019 ceph-mon[88137]: failed to encode map e125 with expected crc 2024-02-07T12:52:31.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:30 smithi019 ceph-mon[88137]: osdmap e125: 8 total, 8 up, 8 in 2024-02-07T12:52:31.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:30 smithi019 ceph-mon[88137]: failed to encode map e125 with expected crc 2024-02-07T12:52:31.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:30 smithi019 ceph-mon[82918]: failed to encode map e124 with expected crc 2024-02-07T12:52:31.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:30 smithi019 ceph-mon[82918]: failed to encode map e124 with expected crc 2024-02-07T12:52:31.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:30 smithi019 ceph-mon[82918]: failed to encode map e125 with expected crc 2024-02-07T12:52:31.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:30 smithi019 ceph-mon[82918]: osdmap e125: 8 total, 8 up, 8 in 2024-02-07T12:52:31.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:30 smithi019 ceph-mon[82918]: failed to encode map e125 with expected crc 2024-02-07T12:52:31.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:30 smithi172 ceph-mon[67636]: failed to encode map e124 with expected crc 2024-02-07T12:52:31.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:30 smithi172 ceph-mon[67636]: failed to encode map e124 with expected crc 2024-02-07T12:52:31.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:30 smithi172 ceph-mon[67636]: failed to encode map e125 with expected crc 2024-02-07T12:52:31.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:30 smithi172 ceph-mon[67636]: osdmap e125: 8 total, 8 up, 8 in 2024-02-07T12:52:31.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:30 smithi172 ceph-mon[67636]: failed to encode map e125 with expected crc 2024-02-07T12:52:32.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:31 smithi019 ceph-mon[82918]: pgmap v176: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 168 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 82/627 objects degraded (13.078%) 2024-02-07T12:52:32.249 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:31 smithi019 ceph-mon[82918]: failed to encode map e125 with expected crc 2024-02-07T12:52:32.249 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:31 smithi019 ceph-mon[82918]: failed to encode map e125 with expected crc 2024-02-07T12:52:32.249 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:31 smithi019 ceph-mon[82918]: failed to encode map e125 with expected crc 2024-02-07T12:52:32.249 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:31 smithi019 ceph-mon[82918]: failed to encode map e125 with expected crc 2024-02-07T12:52:32.249 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:31 smithi019 ceph-mon[82918]: failed to encode map e125 with expected crc 2024-02-07T12:52:32.249 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:31 smithi019 ceph-mon[88137]: pgmap v176: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 168 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 82/627 objects degraded (13.078%) 2024-02-07T12:52:32.249 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:31 smithi019 ceph-mon[88137]: failed to encode map e125 with expected crc 2024-02-07T12:52:32.249 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:31 smithi019 ceph-mon[88137]: failed to encode map e125 with expected crc 2024-02-07T12:52:32.249 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:31 smithi019 ceph-mon[88137]: failed to encode map e125 with expected crc 2024-02-07T12:52:32.249 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:31 smithi019 ceph-mon[88137]: failed to encode map e125 with expected crc 2024-02-07T12:52:32.249 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:31 smithi019 ceph-mon[88137]: failed to encode map e125 with expected crc 2024-02-07T12:52:32.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:31 smithi172 ceph-mon[67636]: pgmap v176: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 168 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 82/627 objects degraded (13.078%) 2024-02-07T12:52:32.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:31 smithi172 ceph-mon[67636]: failed to encode map e125 with expected crc 2024-02-07T12:52:32.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:31 smithi172 ceph-mon[67636]: failed to encode map e125 with expected crc 2024-02-07T12:52:32.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:31 smithi172 ceph-mon[67636]: failed to encode map e125 with expected crc 2024-02-07T12:52:32.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:31 smithi172 ceph-mon[67636]: failed to encode map e125 with expected crc 2024-02-07T12:52:32.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:31 smithi172 ceph-mon[67636]: failed to encode map e125 with expected crc 2024-02-07T12:52:34.040 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:52:33] "GET /metrics HTTP/1.1" 200 38183 "" "Prometheus/2.43.0" 2024-02-07T12:52:34.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:33 smithi019 ceph-mon[88137]: pgmap v178: 161 pgs: 15 active+undersized+degraded, 25 active+undersized, 121 active+clean; 585 KiB data, 168 MiB used, 715 GiB / 715 GiB avail; 46/627 objects degraded (7.337%) 2024-02-07T12:52:34.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:33 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:34.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:33 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:34.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:33 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:52:34.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:33 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:52:34.040 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:33 smithi019 ceph-mon[88137]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 176 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:52:34.041 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:33 smithi019 ceph-mon[88137]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 176 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:52:34.041 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:33 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:34.041 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:33 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:52:34.041 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:33 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:52:34.041 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:33 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:52:34.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:33 smithi019 ceph-mon[82918]: pgmap v178: 161 pgs: 15 active+undersized+degraded, 25 active+undersized, 121 active+clean; 585 KiB data, 168 MiB used, 715 GiB / 715 GiB avail; 46/627 objects degraded (7.337%) 2024-02-07T12:52:34.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:33 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:34.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:33 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:34.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:33 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:52:34.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:33 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:52:34.041 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:33 smithi019 ceph-mon[82918]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 176 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:52:34.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:33 smithi019 ceph-mon[82918]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 176 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:52:34.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:33 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:34.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:33 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:52:34.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:33 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:52:34.042 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:33 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:52:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:33 smithi172 ceph-mon[67636]: pgmap v178: 161 pgs: 15 active+undersized+degraded, 25 active+undersized, 121 active+clean; 585 KiB data, 168 MiB used, 715 GiB / 715 GiB avail; 46/627 objects degraded (7.337%) 2024-02-07T12:52:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:33 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:33 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:33 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:52:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:33 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:52:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:33 smithi172 ceph-mon[67636]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 176 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:52:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:33 smithi172 ceph-mon[67636]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 176 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:52:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:33 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:34.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:33 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:52:34.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:33 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:52:34.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:33 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:52:35.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-mon[88137]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:52:35.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-mon[88137]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:52:35.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-mon[88137]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 46/627 objects degraded (7.337%), 15 pgs degraded) 2024-02-07T12:52:35.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:52:35.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:35.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:35.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:35.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-02-07T12:52:35.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-mon[82918]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:52:35.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-mon[82918]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:52:35.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-mon[82918]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 46/627 objects degraded (7.337%), 15 pgs degraded) 2024-02-07T12:52:35.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:52:35.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:35.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:35.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:35.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-02-07T12:52:35.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:52:34.888+0000 7fb71d671700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:52:35.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:52:35.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:52:35.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:52:35.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:52:35.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:35.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:35.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:52:35.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:52:35.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:52:35.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:52:35.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:52:35.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:52:35.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:52:35.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:52:35.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:52:35.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:52:35.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:52:35.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:52:35.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:35.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:35.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:52:35.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:52:35.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:52:35.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:52:35.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:52:35.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:52:35.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:52:35.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:52:35.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:52:35.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:52:35.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:52:35.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:52:35.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:52:35.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:52:35.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:52:35.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:52:35.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:52:35.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:52:35.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:52:35.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:52:35.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:52:35.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:52:35.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:35.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:35.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:52:35.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:52:35.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:52:35.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:52:35.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:52:35.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:52:35.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:52:35.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:52:35.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:52:35.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:52:35.100 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:52:35.100 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:52:35.100 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:35.100 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:34 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:34 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:52:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:34 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:52:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:34 smithi172 ceph-mon[67636]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 46/627 objects degraded (7.337%), 15 pgs degraded) 2024-02-07T12:52:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:52:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:34 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-02-07T12:52:36.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:35 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:52:36.119 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:52:36.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:52:36.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:52:36.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:52:36.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:36.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:36.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:52:36.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:52:36.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:52:36.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:52:36.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:52:36.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:52:36.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:52:36.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:52:36.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:52:36.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:52:36.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:52:36.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:52:36.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:36.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:36.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:52:36.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:52:36.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:52:36.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:52:36.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:52:36.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:52:36.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:52:36.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:52:36.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:52:36.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:52:36.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:52:36.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:52:36.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:52:36.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:52:36.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:52:36.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:52:36.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:52:36.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:52:36.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:52:36.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:52:36.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:52:36.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:52:36.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:36.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:36.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:52:36.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:52:36.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:52:36.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:52:36.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:52:36.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:52:36.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:52:36.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:52:36.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:52:36.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:52:36.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:52:36.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:52:36.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:36.124 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:36.124 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:35 smithi172 ceph-mon[67636]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-02-07T12:52:36.124 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:35 smithi172 ceph-mon[67636]: Upgrade: osd.4 is safe to restart 2024-02-07T12:52:36.124 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:35 smithi172 ceph-mon[67636]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 176 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:52:36.124 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:35 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:36.124 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:35 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:52:36.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:35 smithi019 ceph-mon[82918]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:52:36.344 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:52:36.344 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:52:36.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:52:36.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:52:36.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:36.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:36.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:52:36.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:52:36.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:52:36.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:52:36.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:52:36.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:52:36.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:52:36.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:52:36.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:52:36.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:52:36.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:52:36.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:52:36.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:36.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:36.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:52:36.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:52:36.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:52:36.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:52:36.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:52:36.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:52:36.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:52:36.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:52:36.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:52:36.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:52:36.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:52:36.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:52:36.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:52:36.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:52:36.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:52:36.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:52:36.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:52:36.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:52:36.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:52:36.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:52:36.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:52:36.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:52:36.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:36.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:36.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:52:36.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:52:36.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:52:36.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:52:36.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:52:36.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:52:36.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:52:36.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:52:36.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:52:36.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:35 smithi019 ceph-mon[88137]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:52:36.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:52:36.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:52:36.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:52:36.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:52:36.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:36.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:36.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:52:36.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:52:36.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:52:36.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:52:36.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:52:36.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:52:36.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:52:36.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:52:36.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:52:36.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:52:36.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:52:36.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:52:36.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:36.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:36.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:52:36.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:52:36.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:52:36.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:52:36.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:52:36.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:52:36.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:52:36.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:52:36.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:52:36.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:52:36.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:52:36.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:52:36.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:52:36.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:52:36.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:52:36.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:52:36.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:52:36.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:52:36.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:52:36.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:52:36.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:52:36.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:52:36.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:36.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:36.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:52:36.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:52:36.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:52:36.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:52:36.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:52:36.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:52:36.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:52:36.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:52:36.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:52:36.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:52:36.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:52:36.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:52:36.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:36.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:36.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:35 smithi019 ceph-mon[82918]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-02-07T12:52:36.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:35 smithi019 ceph-mon[82918]: Upgrade: osd.4 is safe to restart 2024-02-07T12:52:36.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:35 smithi019 ceph-mon[82918]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 176 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:52:36.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:35 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:36.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:35 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:52:36.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:52:36.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:52:36.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:52:36.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:36.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:36.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:35 smithi019 ceph-mon[88137]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-02-07T12:52:36.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:35 smithi019 ceph-mon[88137]: Upgrade: osd.4 is safe to restart 2024-02-07T12:52:36.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:35 smithi019 ceph-mon[88137]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 176 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:52:36.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:35 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:36.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:35 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:52:37.754 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:37 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:52:37.755 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:37 smithi172 ceph-mon[67636]: Upgrade: Updating osd.4 2024-02-07T12:52:37.755 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:37.755 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-02-07T12:52:37.755 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:52:37.755 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:37 smithi172 ceph-mon[67636]: Deploying daemon osd.4 on smithi172 2024-02-07T12:52:37.755 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:37 smithi172 ceph-mon[67636]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 176 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:52:37.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:37 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:52:37.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:37 smithi019 ceph-mon[82918]: Upgrade: Updating osd.4 2024-02-07T12:52:37.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:37.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-02-07T12:52:37.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:52:37.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:37 smithi019 ceph-mon[82918]: Deploying daemon osd.4 on smithi172 2024-02-07T12:52:37.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:37 smithi019 ceph-mon[82918]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 176 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:52:37.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:37 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:52:37.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:37 smithi019 ceph-mon[88137]: Upgrade: Updating osd.4 2024-02-07T12:52:37.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:37.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-02-07T12:52:37.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:52:37.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:37 smithi019 ceph-mon[88137]: Deploying daemon osd.4 on smithi172 2024-02-07T12:52:37.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:37 smithi019 ceph-mon[88137]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 176 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:52:39.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:39 smithi172 ceph-mon[67636]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 176 MiB used, 715 GiB / 715 GiB avail; 119 B/s rd, 0 op/s 2024-02-07T12:52:39.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:39 smithi019 ceph-mon[82918]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 176 MiB used, 715 GiB / 715 GiB avail; 119 B/s rd, 0 op/s 2024-02-07T12:52:39.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:39 smithi019 ceph-mon[88137]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 176 MiB used, 715 GiB / 715 GiB avail; 119 B/s rd, 0 op/s 2024-02-07T12:52:40.119 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:39 smithi172 systemd[1]: Stopping Ceph osd.4 for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:52:40.119 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:40 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4[28829]: 2024-02-07T12:52:40.098+0000 7fb3e66dd700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:52:40.119 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:40 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4[28829]: 2024-02-07T12:52:40.098+0000 7fb3e66dd700 -1 osd.4 125 *** Got signal Terminated *** 2024-02-07T12:52:40.119 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:40 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4[28829]: 2024-02-07T12:52:40.098+0000 7fb3e66dd700 -1 osd.4 125 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:52:41.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:40 smithi019 ceph-mon[88137]: osd.4 marked itself down and dead 2024-02-07T12:52:41.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:41.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:40 smithi019 ceph-mon[82918]: osd.4 marked itself down and dead 2024-02-07T12:52:41.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:41.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:40 smithi172 ceph-mon[67636]: osd.4 marked itself down and dead 2024-02-07T12:52:41.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:41.759 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:41 smithi172 podman[77329]: 2024-02-07 12:52:41.463434998 +0000 UTC m=+1.459881845 container died df72cfd69b3bd8c83aa4a1b9cb1f2dcc352db449e5e459821e34af1fcea63de3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , distribution-scope=public, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, version=8, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, ceph=True, vendor=Red Hat, Inc., RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, GIT_CLEAN=True, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8) 2024-02-07T12:52:42.059 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:41 smithi172 podman[77329]: 2024-02-07 12:52:41.873557698 +0000 UTC m=+1.870004548 container cleanup df72cfd69b3bd8c83aa4a1b9cb1f2dcc352db449e5e459821e34af1fcea63de3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, vendor=Red Hat, Inc., io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , version=8, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, com.redhat.component=centos-stream-container, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, distribution-scope=public, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64) 2024-02-07T12:52:42.060 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:41 smithi172 bash[77329]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4 2024-02-07T12:52:42.060 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:41 smithi172 ceph-mon[67636]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:52:42.060 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:41 smithi172 ceph-mon[67636]: failed to encode map e126 with expected crc 2024-02-07T12:52:42.060 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:41 smithi172 ceph-mon[67636]: failed to encode map e126 with expected crc 2024-02-07T12:52:42.060 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:41 smithi172 ceph-mon[67636]: failed to encode map e126 with expected crc 2024-02-07T12:52:42.060 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:41 smithi172 ceph-mon[67636]: osdmap e126: 8 total, 7 up, 8 in 2024-02-07T12:52:42.060 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:41 smithi172 ceph-mon[67636]: failed to encode map e126 with expected crc 2024-02-07T12:52:42.060 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:41 smithi172 ceph-mon[67636]: failed to encode map e126 with expected crc 2024-02-07T12:52:42.060 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:41 smithi172 ceph-mon[67636]: pgmap v185: 161 pgs: 23 stale+active+clean, 138 active+clean; 585 KiB data, 176 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-02-07T12:52:42.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:41 smithi019 ceph-mon[88137]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:52:42.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:41 smithi019 ceph-mon[88137]: failed to encode map e126 with expected crc 2024-02-07T12:52:42.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:41 smithi019 ceph-mon[88137]: failed to encode map e126 with expected crc 2024-02-07T12:52:42.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:41 smithi019 ceph-mon[88137]: failed to encode map e126 with expected crc 2024-02-07T12:52:42.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:41 smithi019 ceph-mon[88137]: osdmap e126: 8 total, 7 up, 8 in 2024-02-07T12:52:42.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:41 smithi019 ceph-mon[88137]: failed to encode map e126 with expected crc 2024-02-07T12:52:42.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:41 smithi019 ceph-mon[88137]: failed to encode map e126 with expected crc 2024-02-07T12:52:42.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:41 smithi019 ceph-mon[88137]: pgmap v185: 161 pgs: 23 stale+active+clean, 138 active+clean; 585 KiB data, 176 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-02-07T12:52:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:41 smithi019 ceph-mon[82918]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:52:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:41 smithi019 ceph-mon[82918]: failed to encode map e126 with expected crc 2024-02-07T12:52:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:41 smithi019 ceph-mon[82918]: failed to encode map e126 with expected crc 2024-02-07T12:52:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:41 smithi019 ceph-mon[82918]: failed to encode map e126 with expected crc 2024-02-07T12:52:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:41 smithi019 ceph-mon[82918]: osdmap e126: 8 total, 7 up, 8 in 2024-02-07T12:52:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:41 smithi019 ceph-mon[82918]: failed to encode map e126 with expected crc 2024-02-07T12:52:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:41 smithi019 ceph-mon[82918]: failed to encode map e126 with expected crc 2024-02-07T12:52:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:41 smithi019 ceph-mon[82918]: pgmap v185: 161 pgs: 23 stale+active+clean, 138 active+clean; 585 KiB data, 176 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-02-07T12:52:42.369 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:42 smithi172 podman[77357]: 2024-02-07 12:52:42.058193506 +0000 UTC m=+0.595749720 container remove df72cfd69b3bd8c83aa4a1b9cb1f2dcc352db449e5e459821e34af1fcea63de3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, com.redhat.component=centos-stream-container, name=centos-stream, ceph=True, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=) 2024-02-07T12:52:42.813 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:42 smithi172 podman[77453]: 2024-02-07 12:52:42.382251965 +0000 UTC m=+0.017948022 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:52:42.814 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:42 smithi172 podman[77453]: 2024-02-07 12:52:42.567792893 +0000 UTC m=+0.203488949 container create a1314ce59df23f6cdd4b21f4135e18d7827df8e6a5b89722ba8d8d35c83eee02 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4-deactivate, org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-02-07T12:52:43.107 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:42 smithi172 ceph-mon[67636]: failed to encode map e127 with expected crc 2024-02-07T12:52:43.108 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:42 smithi172 ceph-mon[67636]: failed to encode map e127 with expected crc 2024-02-07T12:52:43.108 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:42 smithi172 ceph-mon[67636]: osdmap e127: 8 total, 7 up, 8 in 2024-02-07T12:52:43.108 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:42 smithi172 ceph-mon[67636]: failed to encode map e127 with expected crc 2024-02-07T12:52:43.108 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:42 smithi172 ceph-mon[67636]: failed to encode map e127 with expected crc 2024-02-07T12:52:43.108 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:42 smithi172 podman[77453]: 2024-02-07 12:52:42.812239034 +0000 UTC m=+0.447935097 container init a1314ce59df23f6cdd4b21f4135e18d7827df8e6a5b89722ba8d8d35c83eee02 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4-deactivate, org.label-schema.build-date=20240131, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-02-07T12:52:43.108 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:42 smithi172 podman[77453]: 2024-02-07 12:52:42.815576402 +0000 UTC m=+0.451272455 container start a1314ce59df23f6cdd4b21f4135e18d7827df8e6a5b89722ba8d8d35c83eee02 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4-deactivate, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240131, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.2) 2024-02-07T12:52:43.108 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:42 smithi172 podman[77453]: 2024-02-07 12:52:42.903996019 +0000 UTC m=+0.539692079 container attach a1314ce59df23f6cdd4b21f4135e18d7827df8e6a5b89722ba8d8d35c83eee02 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131) 2024-02-07T12:52:43.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:42 smithi019 ceph-mon[88137]: failed to encode map e127 with expected crc 2024-02-07T12:52:43.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:42 smithi019 ceph-mon[88137]: failed to encode map e127 with expected crc 2024-02-07T12:52:43.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:42 smithi019 ceph-mon[88137]: osdmap e127: 8 total, 7 up, 8 in 2024-02-07T12:52:43.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:42 smithi019 ceph-mon[88137]: failed to encode map e127 with expected crc 2024-02-07T12:52:43.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:42 smithi019 ceph-mon[88137]: failed to encode map e127 with expected crc 2024-02-07T12:52:43.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:42 smithi019 ceph-mon[82918]: failed to encode map e127 with expected crc 2024-02-07T12:52:43.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:42 smithi019 ceph-mon[82918]: failed to encode map e127 with expected crc 2024-02-07T12:52:43.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:42 smithi019 ceph-mon[82918]: osdmap e127: 8 total, 7 up, 8 in 2024-02-07T12:52:43.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:42 smithi019 ceph-mon[82918]: failed to encode map e127 with expected crc 2024-02-07T12:52:43.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:42 smithi019 ceph-mon[82918]: failed to encode map e127 with expected crc 2024-02-07T12:52:43.369 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:43 smithi172 podman[77453]: 2024-02-07 12:52:43.105726593 +0000 UTC m=+0.741422635 container died a1314ce59df23f6cdd4b21f4135e18d7827df8e6a5b89722ba8d8d35c83eee02 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4-deactivate, org.label-schema.build-date=20240131, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , io.buildah.version=1.33.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-02-07T12:52:43.569 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:52:44.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:43 smithi172 ceph-mon[67636]: pgmap v187: 161 pgs: 8 active+undersized+degraded, 31 peering, 20 active+undersized, 102 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail; 24/627 objects degraded (3.828%) 2024-02-07T12:52:44.119 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:43 smithi172 podman[77479]: 2024-02-07 12:52:43.699296666 +0000 UTC m=+0.587034428 container remove a1314ce59df23f6cdd4b21f4135e18d7827df8e6a5b89722ba8d8d35c83eee02 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4-deactivate, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.build-date=20240131, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_CLEAN=True) 2024-02-07T12:52:44.120 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:43 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.4.service: Deactivated successfully. 2024-02-07T12:52:44.120 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:43 smithi172 systemd[1]: Stopped Ceph osd.4 for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:52:44.120 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:43 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.4.service: Consumed 16.515s CPU time. 2024-02-07T12:52:44.120 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:43 smithi172 systemd[1]: Starting Ceph osd.4 for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:52:44.238 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:52:44.239 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (7m) 16s ago 28m 27.3M - 0.25.0 c8568f914cd2 fec9255f35e7 2024-02-07T12:52:44.239 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (6m) 4m ago 28m 82.9M - 9.4.12 a0ea5e3c46d9 aaffac63bc28 2024-02-07T12:52:44.239 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 unknown 16s ago 28m 63.5M - 3.5 e1d6a67b021e 5113aa77e3f2 2024-02-07T12:52:44.239 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283,8765 running (8m) 4m ago 33m 447M - 19.0.0-1252-g44458cce 79957b372307 5aa0c04fdfd7 2024-02-07T12:52:44.239 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (18m) 16s ago 34m 509M - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:52:44.239 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (4m) 16s ago 35m 53.6M 2048M 19.0.0-1252-g44458cce 79957b372307 8b1b0e21b1c5 2024-02-07T12:52:44.239 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (5m) 4m ago 33m 37.0M 2048M 19.0.0-1252-g44458cce 79957b372307 00f454bbac01 2024-02-07T12:52:44.239 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (4m) 16s ago 33m 43.3M 2048M 19.0.0-1252-g44458cce 79957b372307 6214003f6e08 2024-02-07T12:52:44.239 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (7m) 16s ago 29m 16.0M - 1.5.0 0da6a335fe13 c4bdcb1c4d49 2024-02-07T12:52:44.239 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (7m) 4m ago 29m 16.3M - 1.5.0 0da6a335fe13 04ea81f79d85 2024-02-07T12:52:44.239 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (2m) 16s ago 33m 56.0M 2993M 19.0.0-1252-g44458cce 79957b372307 78eddd34a903 2024-02-07T12:52:44.239 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (2m) 16s ago 32m 44.0M 2993M 19.0.0-1252-g44458cce 79957b372307 c37447e76c38 2024-02-07T12:52:44.239 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (75s) 16s ago 32m 50.2M 2993M 19.0.0-1252-g44458cce 79957b372307 90352897da85 2024-02-07T12:52:44.239 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (20s) 16s ago 31m 14.9M 2993M 19.0.0-1252-g44458cce 79957b372307 067cd6b3fd2f 2024-02-07T12:52:44.240 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (31m) 4m ago 31m 55.7M 3248M 17.2.0 e1d6a67b021e df72cfd69b3b 2024-02-07T12:52:44.240 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (31m) 4m ago 31m 52.4M 3248M 17.2.0 e1d6a67b021e 3bdb22c07537 2024-02-07T12:52:44.240 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (30m) 4m ago 30m 52.1M 3248M 17.2.0 e1d6a67b021e 7859b651e0d9 2024-02-07T12:52:44.240 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (30m) 4m ago 30m 55.5M 3248M 17.2.0 e1d6a67b021e b301e7c75d73 2024-02-07T12:52:44.240 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (7m) 4m ago 29m 61.0M - 2.43.0 a07b618ecd1d c9b607eea1be 2024-02-07T12:52:44.240 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (28m) 16s ago 28m 80.2M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:52:44.240 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (28m) 4m ago 28m 78.3M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:52:44.240 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:52:43] "GET /metrics HTTP/1.1" 200 38188 "" "Prometheus/2.43.0" 2024-02-07T12:52:44.241 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:43 smithi019 ceph-mon[88137]: pgmap v187: 161 pgs: 8 active+undersized+degraded, 31 peering, 20 active+undersized, 102 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail; 24/627 objects degraded (3.828%) 2024-02-07T12:52:44.241 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:43 smithi019 ceph-mon[82918]: pgmap v187: 161 pgs: 8 active+undersized+degraded, 31 peering, 20 active+undersized, 102 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail; 24/627 objects degraded (3.828%) 2024-02-07T12:52:44.463 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:44 smithi172 podman[77588]: 2024-02-07 12:52:44.03724906 +0000 UTC m=+0.017972070 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:52:44.463 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:44 smithi172 podman[77588]: 2024-02-07 12:52:44.183849529 +0000 UTC m=+0.164572534 container create 4e936af843d9c479e493b69363605e0970f96678f24d638d1a02eefedceb6649 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240131, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:52:44.463 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:44 smithi172 podman[77588]: 2024-02-07 12:52:44.377897867 +0000 UTC m=+0.358620873 container init 4e936af843d9c479e493b69363605e0970f96678f24d638d1a02eefedceb6649 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4-activate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS) 2024-02-07T12:52:44.463 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:44 smithi172 podman[77588]: 2024-02-07 12:52:44.381012298 +0000 UTC m=+0.361735312 container start 4e936af843d9c479e493b69363605e0970f96678f24d638d1a02eefedceb6649 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4-activate, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240131) 2024-02-07T12:52:44.643 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:52:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: "mon": { 2024-02-07T12:52:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 3 2024-02-07T12:52:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:52:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr": { 2024-02-07T12:52:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:52:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:52:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: "osd": { 2024-02-07T12:52:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-02-07T12:52:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 4 2024-02-07T12:52:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:52:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw": { 2024-02-07T12:52:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:52:44.644 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:52:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: "overall": { 2024-02-07T12:52:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-02-07T12:52:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 9 2024-02-07T12:52:44.645 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:52:44.645 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:52:44.823 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:44 smithi172 podman[77588]: 2024-02-07 12:52:44.461440156 +0000 UTC m=+0.442163174 container attach 4e936af843d9c479e493b69363605e0970f96678f24d638d1a02eefedceb6649 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4-activate, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, GIT_CLEAN=True) 2024-02-07T12:52:44.984 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:52:44.984 INFO:teuthology.orchestra.run.smithi019.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:52:44.984 INFO:teuthology.orchestra.run.smithi019.stdout: "in_progress": true, 2024-02-07T12:52:44.984 INFO:teuthology.orchestra.run.smithi019.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-02-07T12:52:44.984 INFO:teuthology.orchestra.run.smithi019.stdout: "services_complete": [], 2024-02-07T12:52:44.984 INFO:teuthology.orchestra.run.smithi019.stdout: "progress": "4/8 daemons upgraded", 2024-02-07T12:52:44.984 INFO:teuthology.orchestra.run.smithi019.stdout: "message": "Currently upgrading osd daemons", 2024-02-07T12:52:44.984 INFO:teuthology.orchestra.run.smithi019.stdout: "is_paused": false 2024-02-07T12:52:44.984 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:52:45.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:44 smithi172 ceph-mon[67636]: from='client.44472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:45.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:44 smithi172 ceph-mon[67636]: from='client.44478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:45.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:44 smithi172 ceph-mon[67636]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-02-07T12:52:45.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:44 smithi172 ceph-mon[67636]: Health check failed: Degraded data redundancy: 24/627 objects degraded (3.828%), 8 pgs degraded (PG_DEGRADED) 2024-02-07T12:52:45.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:44 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/2643358912' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:45.119 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:44 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4-activate[77602]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-02-07T12:52:45.120 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:44 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4-activate[77602]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-02-07T12:52:45.120 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:44 smithi172 bash[77588]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-02-07T12:52:45.120 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:44 smithi172 bash[77588]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-02-07T12:52:45.120 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:44 smithi172 bash[77588]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-02-07T12:52:45.120 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:44 smithi172 bash[77588]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-02-07T12:52:45.120 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:44 smithi172 bash[77588]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-02-07T12:52:45.120 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:44 smithi172 bash[77588]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-02-07T12:52:45.120 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:44 smithi172 bash[77588]: --> ceph-volume raw activate successful for osd ID: 4 2024-02-07T12:52:45.120 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:44 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4-activate[77602]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-02-07T12:52:45.120 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:44 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4-activate[77602]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-02-07T12:52:45.120 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:44 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4-activate[77602]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-02-07T12:52:45.120 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:44 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4-activate[77602]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-02-07T12:52:45.120 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:44 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4-activate[77602]: --> ceph-volume raw activate successful for osd ID: 4 2024-02-07T12:52:45.120 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:44 smithi172 podman[77588]: 2024-02-07 12:52:44.846213649 +0000 UTC m=+0.826936657 container died 4e936af843d9c479e493b69363605e0970f96678f24d638d1a02eefedceb6649 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4-activate, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-02-07T12:52:45.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:44 smithi019 ceph-mon[88137]: from='client.44472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:45.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:44 smithi019 ceph-mon[88137]: from='client.44478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:45.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:44 smithi019 ceph-mon[88137]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-02-07T12:52:45.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:44 smithi019 ceph-mon[88137]: Health check failed: Degraded data redundancy: 24/627 objects degraded (3.828%), 8 pgs degraded (PG_DEGRADED) 2024-02-07T12:52:45.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:44 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/2643358912' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:45.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:44 smithi019 ceph-mon[82918]: from='client.44472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:45.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:44 smithi019 ceph-mon[82918]: from='client.44478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:45.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:44 smithi019 ceph-mon[82918]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-02-07T12:52:45.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:44 smithi019 ceph-mon[82918]: Health check failed: Degraded data redundancy: 24/627 objects degraded (3.828%), 8 pgs degraded (PG_DEGRADED) 2024-02-07T12:52:45.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:44 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/2643358912' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:45.726 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:45 smithi172 podman[77734]: 2024-02-07 12:52:45.473362249 +0000 UTC m=+0.620480948 container remove 4e936af843d9c479e493b69363605e0970f96678f24d638d1a02eefedceb6649 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4-activate, io.buildah.version=1.33.2, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-02-07T12:52:45.726 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:45 smithi172 podman[77802]: 2024-02-07 12:52:45.590476629 +0000 UTC m=+0.017833252 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:52:45.726 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:45 smithi172 podman[77802]: 2024-02-07 12:52:45.724460502 +0000 UTC m=+0.151817171 container create dbb47a8ad5ac9544a15d786f62a16c1bccaa461086e55aa04ea36b297cecf43d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:52:45.977 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:45 smithi172 ceph-mon[67636]: from='client.44484 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:45.977 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:45 smithi172 ceph-mon[67636]: from='client.44496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:45.977 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:45 smithi172 ceph-mon[67636]: pgmap v188: 161 pgs: 8 active+undersized+degraded, 31 peering, 20 active+undersized, 102 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail; 24/627 objects degraded (3.828%) 2024-02-07T12:52:45.977 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:45 smithi172 podman[77802]: 2024-02-07 12:52:45.952132194 +0000 UTC m=+0.379488831 container init dbb47a8ad5ac9544a15d786f62a16c1bccaa461086e55aa04ea36b297cecf43d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:52:45.977 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:45 smithi172 podman[77802]: 2024-02-07 12:52:45.955487769 +0000 UTC m=+0.382844393 container start dbb47a8ad5ac9544a15d786f62a16c1bccaa461086e55aa04ea36b297cecf43d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4, io.buildah.version=1.33.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:52:46.213 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:45 smithi019 ceph-mon[88137]: from='client.44484 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:46.213 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:45 smithi019 ceph-mon[88137]: from='client.44496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:46.213 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:45 smithi019 ceph-mon[88137]: pgmap v188: 161 pgs: 8 active+undersized+degraded, 31 peering, 20 active+undersized, 102 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail; 24/627 objects degraded (3.828%) 2024-02-07T12:52:46.213 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:45 smithi019 ceph-mon[82918]: from='client.44484 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:46.213 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:45 smithi019 ceph-mon[82918]: from='client.44496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:52:46.213 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:45 smithi019 ceph-mon[82918]: pgmap v188: 161 pgs: 8 active+undersized+degraded, 31 peering, 20 active+undersized, 102 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail; 24/627 objects degraded (3.828%) 2024-02-07T12:52:46.369 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:45 smithi172 ceph-osd[77822]: -- 172.21.15.172:0/398512408 <== mon.2 v2:172.21.15.172:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5566170dd6c0 con 0x5566170aec00 2024-02-07T12:52:46.369 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:46 smithi172 bash[77802]: dbb47a8ad5ac9544a15d786f62a16c1bccaa461086e55aa04ea36b297cecf43d 2024-02-07T12:52:46.369 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:46 smithi172 systemd[1]: Started Ceph osd.4 for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:52:46.869 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:46 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4[77818]: 2024-02-07T12:52:46.499+0000 7f54061e27c0 -1 Falling back to public interface 2024-02-07T12:52:47.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:47 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:47.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:47 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:47.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:47 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:52:47.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:47 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:52:47.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:47 smithi172 ceph-mon[67636]: pgmap v189: 161 pgs: 26 active+undersized+degraded, 42 active+undersized, 93 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail; 105/627 objects degraded (16.746%) 2024-02-07T12:52:47.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:47 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:47.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:47 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:47.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:47 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:52:47.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:47 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:52:47.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:47 smithi019 ceph-mon[88137]: pgmap v189: 161 pgs: 26 active+undersized+degraded, 42 active+undersized, 93 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail; 105/627 objects degraded (16.746%) 2024-02-07T12:52:47.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:47 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:47.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:47 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:47.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:47 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:52:47.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:47 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:52:47.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:47 smithi019 ceph-mon[82918]: pgmap v189: 161 pgs: 26 active+undersized+degraded, 42 active+undersized, 93 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail; 105/627 objects degraded (16.746%) 2024-02-07T12:52:48.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:48 smithi172 ceph-mon[67636]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-02-07T12:52:48.516 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:48 smithi019 ceph-mon[88137]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-02-07T12:52:48.516 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:48 smithi019 ceph-mon[82918]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-02-07T12:52:49.369 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:49 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4[77818]: 2024-02-07T12:52:49.187+0000 7f54061e27c0 -1 osd.4 0 read_superblock omap replica is missing. 2024-02-07T12:52:49.369 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:49 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4[77818]: 2024-02-07T12:52:49.217+0000 7f54061e27c0 -1 osd.4 125 log_to_monitors true 2024-02-07T12:52:50.369 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:52:50 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4[77818]: 2024-02-07T12:52:50.142+0000 7f53f7641700 -1 osd.4 125 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:52:50.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:50 smithi172 ceph-mon[67636]: pgmap v190: 161 pgs: 26 active+undersized+degraded, 42 active+undersized, 93 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail; 105/627 objects degraded (16.746%) 2024-02-07T12:52:50.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:50 smithi172 ceph-mon[67636]: from='osd.4 [v2:172.21.15.172:6800/2006496835,v1:172.21.15.172:6801/2006496835]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-02-07T12:52:50.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:50 smithi172 ceph-mon[67636]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-02-07T12:52:50.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:50 smithi172 ceph-mon[67636]: Health check update: Degraded data redundancy: 105/627 objects degraded (16.746%), 26 pgs degraded (PG_DEGRADED) 2024-02-07T12:52:50.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:50 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:50.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:50 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:52:50.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:50 smithi019 ceph-mon[88137]: pgmap v190: 161 pgs: 26 active+undersized+degraded, 42 active+undersized, 93 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail; 105/627 objects degraded (16.746%) 2024-02-07T12:52:50.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:50 smithi019 ceph-mon[88137]: from='osd.4 [v2:172.21.15.172:6800/2006496835,v1:172.21.15.172:6801/2006496835]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-02-07T12:52:50.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:50 smithi019 ceph-mon[88137]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-02-07T12:52:50.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:50 smithi019 ceph-mon[88137]: Health check update: Degraded data redundancy: 105/627 objects degraded (16.746%), 26 pgs degraded (PG_DEGRADED) 2024-02-07T12:52:50.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:50 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:50.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:50 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:52:50.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:50 smithi019 ceph-mon[82918]: pgmap v190: 161 pgs: 26 active+undersized+degraded, 42 active+undersized, 93 active+clean; 585 KiB data, 177 MiB used, 715 GiB / 715 GiB avail; 105/627 objects degraded (16.746%) 2024-02-07T12:52:50.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:50 smithi019 ceph-mon[82918]: from='osd.4 [v2:172.21.15.172:6800/2006496835,v1:172.21.15.172:6801/2006496835]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-02-07T12:52:50.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:50 smithi019 ceph-mon[82918]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-02-07T12:52:50.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:50 smithi019 ceph-mon[82918]: Health check update: Degraded data redundancy: 105/627 objects degraded (16.746%), 26 pgs degraded (PG_DEGRADED) 2024-02-07T12:52:50.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:50 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:50.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:50 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:52:51.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:51 smithi019 ceph-mon[88137]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-02-07T12:52:51.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:51 smithi019 ceph-mon[88137]: failed to encode map e128 with expected crc 2024-02-07T12:52:51.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:51 smithi019 ceph-mon[88137]: from='osd.4 [v2:172.21.15.172:6800/2006496835,v1:172.21.15.172:6801/2006496835]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:52:51.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:51 smithi019 ceph-mon[88137]: failed to encode map e128 with expected crc 2024-02-07T12:52:51.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:51 smithi019 ceph-mon[88137]: osdmap e128: 8 total, 7 up, 8 in 2024-02-07T12:52:51.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:51 smithi019 ceph-mon[88137]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:52:51.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:51 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:51.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:51 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:51.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:51 smithi019 ceph-mon[82918]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-02-07T12:52:51.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:51 smithi019 ceph-mon[82918]: failed to encode map e128 with expected crc 2024-02-07T12:52:51.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:51 smithi019 ceph-mon[82918]: from='osd.4 [v2:172.21.15.172:6800/2006496835,v1:172.21.15.172:6801/2006496835]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:52:51.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:51 smithi019 ceph-mon[82918]: failed to encode map e128 with expected crc 2024-02-07T12:52:51.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:51 smithi019 ceph-mon[82918]: osdmap e128: 8 total, 7 up, 8 in 2024-02-07T12:52:51.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:51 smithi019 ceph-mon[82918]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:52:51.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:51 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:51.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:51 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:51.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:51 smithi172 ceph-mon[67636]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-02-07T12:52:51.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:51 smithi172 ceph-mon[67636]: failed to encode map e128 with expected crc 2024-02-07T12:52:51.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:51 smithi172 ceph-mon[67636]: from='osd.4 [v2:172.21.15.172:6800/2006496835,v1:172.21.15.172:6801/2006496835]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:52:51.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:51 smithi172 ceph-mon[67636]: failed to encode map e128 with expected crc 2024-02-07T12:52:51.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:51 smithi172 ceph-mon[67636]: osdmap e128: 8 total, 7 up, 8 in 2024-02-07T12:52:51.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:51 smithi172 ceph-mon[67636]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:52:51.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:51 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:51.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:51 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:52.258 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:52 smithi172 ceph-mon[67636]: pgmap v192: 161 pgs: 26 active+undersized+degraded, 42 active+undersized, 93 active+clean; 585 KiB data, 192 MiB used, 715 GiB / 715 GiB avail; 105/627 objects degraded (16.746%) 2024-02-07T12:52:52.260 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:52 smithi172 ceph-mon[67636]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:52:52.260 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:52 smithi172 ceph-mon[67636]: failed to encode map e129 with expected crc 2024-02-07T12:52:52.260 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:52 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:52:52.260 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:52 smithi172 ceph-mon[67636]: failed to encode map e129 with expected crc 2024-02-07T12:52:52.260 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:52 smithi172 ceph-mon[67636]: osd.4 [v2:172.21.15.172:6800/2006496835,v1:172.21.15.172:6801/2006496835] boot 2024-02-07T12:52:52.260 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:52 smithi172 ceph-mon[67636]: osdmap e129: 8 total, 8 up, 8 in 2024-02-07T12:52:52.260 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:52 smithi172 ceph-mon[67636]: failed to encode map e128 with expected crc 2024-02-07T12:52:52.260 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:52 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:52.260 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:52 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:52.261 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:52 smithi172 ceph-mon[67636]: osdmap e130: 8 total, 8 up, 8 in 2024-02-07T12:52:52.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:52 smithi019 ceph-mon[82918]: pgmap v192: 161 pgs: 26 active+undersized+degraded, 42 active+undersized, 93 active+clean; 585 KiB data, 192 MiB used, 715 GiB / 715 GiB avail; 105/627 objects degraded (16.746%) 2024-02-07T12:52:52.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:52 smithi019 ceph-mon[82918]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:52:52.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:52 smithi019 ceph-mon[82918]: failed to encode map e129 with expected crc 2024-02-07T12:52:52.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:52 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:52:52.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:52 smithi019 ceph-mon[82918]: failed to encode map e129 with expected crc 2024-02-07T12:52:52.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:52 smithi019 ceph-mon[82918]: osd.4 [v2:172.21.15.172:6800/2006496835,v1:172.21.15.172:6801/2006496835] boot 2024-02-07T12:52:52.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:52 smithi019 ceph-mon[82918]: osdmap e129: 8 total, 8 up, 8 in 2024-02-07T12:52:52.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:52 smithi019 ceph-mon[82918]: failed to encode map e128 with expected crc 2024-02-07T12:52:52.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:52 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:52.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:52 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:52.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:52 smithi019 ceph-mon[82918]: osdmap e130: 8 total, 8 up, 8 in 2024-02-07T12:52:52.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:52 smithi019 ceph-mon[88137]: pgmap v192: 161 pgs: 26 active+undersized+degraded, 42 active+undersized, 93 active+clean; 585 KiB data, 192 MiB used, 715 GiB / 715 GiB avail; 105/627 objects degraded (16.746%) 2024-02-07T12:52:52.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:52 smithi019 ceph-mon[88137]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:52:52.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:52 smithi019 ceph-mon[88137]: failed to encode map e129 with expected crc 2024-02-07T12:52:52.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:52 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:52:52.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:52 smithi019 ceph-mon[88137]: failed to encode map e129 with expected crc 2024-02-07T12:52:52.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:52 smithi019 ceph-mon[88137]: osd.4 [v2:172.21.15.172:6800/2006496835,v1:172.21.15.172:6801/2006496835] boot 2024-02-07T12:52:52.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:52 smithi019 ceph-mon[88137]: osdmap e129: 8 total, 8 up, 8 in 2024-02-07T12:52:52.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:52 smithi019 ceph-mon[88137]: failed to encode map e128 with expected crc 2024-02-07T12:52:52.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:52 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:52.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:52 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:52.596 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:52 smithi019 ceph-mon[88137]: osdmap e130: 8 total, 8 up, 8 in 2024-02-07T12:52:53.459 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:53 smithi172 ceph-mon[67636]: failed to encode map e130 with expected crc 2024-02-07T12:52:53.459 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:53 smithi172 ceph-mon[67636]: failed to encode map e130 with expected crc 2024-02-07T12:52:53.459 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:53 smithi172 ceph-mon[67636]: failed to encode map e130 with expected crc 2024-02-07T12:52:53.459 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:53 smithi172 ceph-mon[67636]: failed to encode map e130 with expected crc 2024-02-07T12:52:53.459 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:53 smithi172 ceph-mon[67636]: failed to encode map e130 with expected crc 2024-02-07T12:52:53.459 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:53 smithi172 ceph-mon[67636]: failed to encode map e130 with expected crc 2024-02-07T12:52:53.459 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:53 smithi172 ceph-mon[67636]: failed to encode map e130 with expected crc 2024-02-07T12:52:53.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:53 smithi019 ceph-mon[82918]: failed to encode map e130 with expected crc 2024-02-07T12:52:53.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:53 smithi019 ceph-mon[82918]: failed to encode map e130 with expected crc 2024-02-07T12:52:53.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:53 smithi019 ceph-mon[82918]: failed to encode map e130 with expected crc 2024-02-07T12:52:53.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:53 smithi019 ceph-mon[82918]: failed to encode map e130 with expected crc 2024-02-07T12:52:53.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:53 smithi019 ceph-mon[82918]: failed to encode map e130 with expected crc 2024-02-07T12:52:53.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:53 smithi019 ceph-mon[82918]: failed to encode map e130 with expected crc 2024-02-07T12:52:53.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:53 smithi019 ceph-mon[82918]: failed to encode map e130 with expected crc 2024-02-07T12:52:53.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:53 smithi019 ceph-mon[88137]: failed to encode map e130 with expected crc 2024-02-07T12:52:53.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:53 smithi019 ceph-mon[88137]: failed to encode map e130 with expected crc 2024-02-07T12:52:53.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:53 smithi019 ceph-mon[88137]: failed to encode map e130 with expected crc 2024-02-07T12:52:53.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:53 smithi019 ceph-mon[88137]: failed to encode map e130 with expected crc 2024-02-07T12:52:53.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:53 smithi019 ceph-mon[88137]: failed to encode map e130 with expected crc 2024-02-07T12:52:53.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:53 smithi019 ceph-mon[88137]: failed to encode map e130 with expected crc 2024-02-07T12:52:53.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:53 smithi019 ceph-mon[88137]: failed to encode map e130 with expected crc 2024-02-07T12:52:54.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:54 smithi019 ceph-mon[82918]: pgmap v195: 161 pgs: 13 peering, 18 active+undersized+degraded, 27 active+undersized, 103 active+clean; 585 KiB data, 192 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 76/627 objects degraded (12.121%) 2024-02-07T12:52:54.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:54 smithi019 ceph-mon[88137]: pgmap v195: 161 pgs: 13 peering, 18 active+undersized+degraded, 27 active+undersized, 103 active+clean; 585 KiB data, 192 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 76/627 objects degraded (12.121%) 2024-02-07T12:52:54.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:52:53] "GET /metrics HTTP/1.1" 200 38195 "" "Prometheus/2.43.0" 2024-02-07T12:52:54.453 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:54 smithi172 ceph-mon[67636]: pgmap v195: 161 pgs: 13 peering, 18 active+undersized+degraded, 27 active+undersized, 103 active+clean; 585 KiB data, 192 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 76/627 objects degraded (12.121%) 2024-02-07T12:52:55.921 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:55 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:55.922 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:55 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:55.922 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:55 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:52:55.922 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:55 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:52:55.922 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:55 smithi019 ceph-mon[88137]: pgmap v196: 161 pgs: 13 peering, 18 active+undersized+degraded, 27 active+undersized, 103 active+clean; 585 KiB data, 192 MiB used, 715 GiB / 715 GiB avail; 872 B/s rd, 0 op/s; 76/627 objects degraded (12.121%) 2024-02-07T12:52:55.922 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:55 smithi019 ceph-mon[88137]: pgmap v197: 161 pgs: 13 peering, 18 active+undersized+degraded, 27 active+undersized, 103 active+clean; 585 KiB data, 192 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 76/627 objects degraded (12.121%) 2024-02-07T12:52:55.922 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:55 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:55.922 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:55 smithi019 ceph-mon[88137]: Health check update: Degraded data redundancy: 76/627 objects degraded (12.121%), 18 pgs degraded (PG_DEGRADED) 2024-02-07T12:52:55.922 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:55 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:52:55.922 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:55 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:52:55.922 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:55 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:52:55.922 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:55 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:55.922 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:55 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:55.923 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:55 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:52:55.923 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:55 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:52:55.923 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:55 smithi019 ceph-mon[82918]: pgmap v196: 161 pgs: 13 peering, 18 active+undersized+degraded, 27 active+undersized, 103 active+clean; 585 KiB data, 192 MiB used, 715 GiB / 715 GiB avail; 872 B/s rd, 0 op/s; 76/627 objects degraded (12.121%) 2024-02-07T12:52:55.923 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:55 smithi019 ceph-mon[82918]: pgmap v197: 161 pgs: 13 peering, 18 active+undersized+degraded, 27 active+undersized, 103 active+clean; 585 KiB data, 192 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 76/627 objects degraded (12.121%) 2024-02-07T12:52:55.923 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:55 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:55.923 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:55 smithi019 ceph-mon[82918]: Health check update: Degraded data redundancy: 76/627 objects degraded (12.121%), 18 pgs degraded (PG_DEGRADED) 2024-02-07T12:52:55.923 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:55 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:52:55.923 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:55 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:52:55.923 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:55 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:52:56.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:55 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:56.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:55 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:56.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:55 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:52:56.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:55 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:52:56.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:55 smithi172 ceph-mon[67636]: pgmap v196: 161 pgs: 13 peering, 18 active+undersized+degraded, 27 active+undersized, 103 active+clean; 585 KiB data, 192 MiB used, 715 GiB / 715 GiB avail; 872 B/s rd, 0 op/s; 76/627 objects degraded (12.121%) 2024-02-07T12:52:56.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:55 smithi172 ceph-mon[67636]: pgmap v197: 161 pgs: 13 peering, 18 active+undersized+degraded, 27 active+undersized, 103 active+clean; 585 KiB data, 192 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 76/627 objects degraded (12.121%) 2024-02-07T12:52:56.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:55 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:52:56.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:55 smithi172 ceph-mon[67636]: Health check update: Degraded data redundancy: 76/627 objects degraded (12.121%), 18 pgs degraded (PG_DEGRADED) 2024-02-07T12:52:56.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:55 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:52:56.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:55 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:52:56.120 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:55 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:52:57.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-mon[82918]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:52:57.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-mon[82918]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:52:57.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-mon[88137]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:52:57.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-mon[88137]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:52:57.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:52:56.910+0000 7fb71d671700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:52:57.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:52:57.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:52:57.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:52:57.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:52:57.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:57.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:57.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:52:57.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:52:57.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:52:57.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:52:57.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:52:57.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:52:57.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:52:57.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:52:57.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:52:57.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:52:57.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:52:57.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:52:57.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:57.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:57.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:52:57.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:52:57.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:52:57.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:52:57.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:52:57.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:52:57.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:52:57.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:52:57.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:52:57.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:52:57.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:52:57.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:52:57.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:52:57.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:52:57.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:52:57.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:52:57.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:52:57.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:52:57.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:52:57.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:52:57.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:52:57.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:52:57.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:57.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:57.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:52:57.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:52:57.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:52:57.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:52:57.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:52:57.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:52:57.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:52:57.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:52:57.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:52:57.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:52:57.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:52:57.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:52:57.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:57.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:57.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:52:56 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:52:56.924+0000 7fb74ced9700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-02-07T12:52:57.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:56 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:52:57.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:56 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:52:58.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:57 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:52:58.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:57 smithi172 ceph-mon[67636]: pgmap v198: 161 pgs: 6 peering, 155 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 872 B/s rd, 0 op/s 2024-02-07T12:52:58.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:57 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:52:58.119 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:52:58.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:52:58.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:52:58.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:52:58.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:58.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:58.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:52:58.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:52:58.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:52:58.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:52:58.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:52:58.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:52:58.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:52:58.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:52:58.120 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:52:58.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:52:58.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:52:58.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:52:58.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:58.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:58.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:52:58.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:52:58.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:52:58.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:52:58.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:52:58.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:52:58.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:52:58.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:52:58.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:52:58.121 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:52:58.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:52:58.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:52:58.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:52:58.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:52:58.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:52:58.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:52:58.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:52:58.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:52:58.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:52:58.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:52:58.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:52:58.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:52:58.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:58.122 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:58.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:52:58.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:52:58.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:52:58.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:52:58.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:52:58.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:52:58.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:52:58.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:52:58.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:52:58.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:52:58.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:52:58.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:52:58.123 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:58.124 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:58.124 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:57 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:52:58.124 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:57 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:58.124 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:57 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:58.124 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:57 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:58.124 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:57 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-07T12:52:58.124 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:57 smithi172 ceph-mon[67636]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-07T12:52:58.124 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:57 smithi172 ceph-mon[67636]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-02-07T12:52:58.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:57 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:52:58.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:57 smithi019 ceph-mon[88137]: pgmap v198: 161 pgs: 6 peering, 155 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 872 B/s rd, 0 op/s 2024-02-07T12:52:58.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:57 smithi019 ceph-mon[88137]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:52:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:52:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:52:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:52:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:52:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:52:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:52:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:52:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:52:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:52:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:52:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:52:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:52:58.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:52:58.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:52:58.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:52:58.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:52:58.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:58.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:58.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:52:58.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:52:58.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:52:58.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:52:58.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:52:58.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:52:58.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:52:58.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:52:58.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:52:58.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:52:58.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:52:58.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:52:58.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:52:58.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:52:58.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:52:58.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:52:58.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:52:58.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:52:58.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:52:58.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:52:58.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:52:58.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:52:58.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:58.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:58.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:52:58.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:52:58.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:52:58.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:52:58.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:52:58.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:52:58.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:52:58.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:52:58.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:57 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:52:58.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:57 smithi019 ceph-mon[82918]: pgmap v198: 161 pgs: 6 peering, 155 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 872 B/s rd, 0 op/s 2024-02-07T12:52:58.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:57 smithi019 ceph-mon[82918]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:52:58.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:52:58.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:52:58.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:52:58.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:52:58.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:58.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:58.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:52:58.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:52:58.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:52:58.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:52:58.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:52:58.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:52:58.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:52:58.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:52:58.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:52:58.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:52:58.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:52:58.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:52:58.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:58.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:58.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:52:58.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:52:58.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:52:58.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:52:58.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:52:58.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:52:58.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:52:58.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:52:58.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:52:58.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:52:58.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:52:58.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:52:58.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:52:58.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:52:58.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:52:58.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:52:58.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:52:58.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:52:58.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:52:58.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:52:58.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:52:58.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:52:58.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:58.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:58.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:52:58.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:52:58.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:52:58.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:52:58.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:52:58.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:52:58.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:52:58.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:52:58.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:52:58.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:52:58.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:52:58.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:52:58.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:58.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:58.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:57 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:52:58.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:57 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:58.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:57 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:58.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:57 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:58.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:57 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-07T12:52:58.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:57 smithi019 ceph-mon[82918]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-07T12:52:58.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:57 smithi019 ceph-mon[82918]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-02-07T12:52:58.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:52:58.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:52:58.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:52:58.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:52:58.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:52:58.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:52:58.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:57 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:52:58.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:57 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:58.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:57 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:58.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:57 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:52:58.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:57 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-07T12:52:58.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:57 smithi019 ceph-mon[88137]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-07T12:52:58.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:57 smithi019 ceph-mon[88137]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-02-07T12:52:59.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:58 smithi172 ceph-mon[67636]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 76/627 objects degraded (12.121%), 18 pgs degraded) 2024-02-07T12:52:59.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:58 smithi019 ceph-mon[88137]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 76/627 objects degraded (12.121%), 18 pgs degraded) 2024-02-07T12:52:59.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:58 smithi019 ceph-mon[82918]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 76/627 objects degraded (12.121%), 18 pgs degraded) 2024-02-07T12:53:00.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:52:59 smithi172 ceph-mon[67636]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-02-07T12:53:00.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:52:59 smithi019 ceph-mon[88137]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-02-07T12:53:00.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:52:59 smithi019 ceph-mon[82918]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-02-07T12:53:02.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:01 smithi019 ceph-mon[88137]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:53:02.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:01 smithi019 ceph-mon[82918]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:53:02.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:01 smithi172 ceph-mon[67636]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:53:04.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:03 smithi019 ceph-mon[88137]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 518 B/s rd, 0 op/s 2024-02-07T12:53:04.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:03 smithi019 ceph-mon[82918]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 518 B/s rd, 0 op/s 2024-02-07T12:53:04.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:53:03] "GET /metrics HTTP/1.1" 200 38195 "" "Prometheus/2.43.0" 2024-02-07T12:53:04.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:03 smithi172 ceph-mon[67636]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 518 B/s rd, 0 op/s 2024-02-07T12:53:06.233 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:05 smithi019 ceph-mon[88137]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:53:06.233 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:06.233 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:53:06.233 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:05 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:06.233 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:05 smithi019 ceph-mon[82918]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:53:06.233 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:06.233 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:53:06.234 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:05 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:05 smithi172 ceph-mon[67636]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:53:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:53:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:05 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:08.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:07 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:53:08.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:07 smithi019 ceph-mon[88137]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:53:08.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:07 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:53:08.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:07 smithi019 ceph-mon[82918]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:53:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:07 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:53:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:07 smithi172 ceph-mon[67636]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:53:10.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:09 smithi019 ceph-mon[82918]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:53:10.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:09 smithi019 ceph-mon[88137]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:53:10.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:09 smithi172 ceph-mon[67636]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:53:12.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:11 smithi019 ceph-mon[88137]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:53:12.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:11 smithi019 ceph-mon[82918]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:53:12.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:11 smithi172 ceph-mon[67636]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:53:13.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:12 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-07T12:53:13.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:12 smithi172 ceph-mon[67636]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-07T12:53:13.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:12 smithi172 ceph-mon[67636]: Upgrade: osd.5 is safe to restart 2024-02-07T12:53:13.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:12 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-07T12:53:13.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:12 smithi019 ceph-mon[88137]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-07T12:53:13.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:12 smithi019 ceph-mon[88137]: Upgrade: osd.5 is safe to restart 2024-02-07T12:53:13.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:12 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-07T12:53:13.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:12 smithi019 ceph-mon[82918]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-07T12:53:13.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:12 smithi019 ceph-mon[82918]: Upgrade: osd.5 is safe to restart 2024-02-07T12:53:14.186 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:13 smithi172 ceph-mon[67636]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:53:14.186 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:13 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:14.186 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:13 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-02-07T12:53:14.186 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:13 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:53:14.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:53:13] "GET /metrics HTTP/1.1" 200 38196 "" "Prometheus/2.43.0" 2024-02-07T12:53:14.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:13 smithi019 ceph-mon[82918]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:53:14.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:13 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:14.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:13 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-02-07T12:53:14.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:13 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:53:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:13 smithi019 ceph-mon[88137]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:53:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:13 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:13 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-02-07T12:53:14.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:13 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:53:15.194 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:14 smithi172 ceph-mon[67636]: Upgrade: Updating osd.5 2024-02-07T12:53:15.194 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:14 smithi172 ceph-mon[67636]: Deploying daemon osd.5 on smithi172 2024-02-07T12:53:15.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:14 smithi019 ceph-mon[82918]: Upgrade: Updating osd.5 2024-02-07T12:53:15.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:14 smithi019 ceph-mon[82918]: Deploying daemon osd.5 on smithi172 2024-02-07T12:53:15.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:14 smithi019 ceph-mon[88137]: Upgrade: Updating osd.5 2024-02-07T12:53:15.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:14 smithi019 ceph-mon[88137]: Deploying daemon osd.5 on smithi172 2024-02-07T12:53:15.350 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:53:16.016 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:53:16.016 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (7m) 47s ago 29m 27.3M - 0.25.0 c8568f914cd2 fec9255f35e7 2024-02-07T12:53:16.016 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (7m) 25s ago 29m 87.1M - 9.4.12 a0ea5e3c46d9 aaffac63bc28 2024-02-07T12:53:16.016 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 unknown 47s ago 28m 63.5M - 3.5 e1d6a67b021e 5113aa77e3f2 2024-02-07T12:53:16.016 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283,8765 running (8m) 25s ago 33m 451M - 19.0.0-1252-g44458cce 79957b372307 5aa0c04fdfd7 2024-02-07T12:53:16.016 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (18m) 47s ago 35m 509M - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:53:16.016 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (5m) 47s ago 35m 53.6M 2048M 19.0.0-1252-g44458cce 79957b372307 8b1b0e21b1c5 2024-02-07T12:53:16.016 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (6m) 25s ago 34m 50.9M 2048M 19.0.0-1252-g44458cce 79957b372307 00f454bbac01 2024-02-07T12:53:16.016 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (4m) 47s ago 34m 43.3M 2048M 19.0.0-1252-g44458cce 79957b372307 6214003f6e08 2024-02-07T12:53:16.016 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (8m) 47s ago 30m 16.0M - 1.5.0 0da6a335fe13 c4bdcb1c4d49 2024-02-07T12:53:16.016 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (8m) 25s ago 30m 16.3M - 1.5.0 0da6a335fe13 04ea81f79d85 2024-02-07T12:53:16.017 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (3m) 47s ago 33m 56.0M 2993M 19.0.0-1252-g44458cce 79957b372307 78eddd34a903 2024-02-07T12:53:16.017 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (2m) 47s ago 33m 44.0M 2993M 19.0.0-1252-g44458cce 79957b372307 c37447e76c38 2024-02-07T12:53:16.017 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (107s) 47s ago 32m 50.2M 2993M 19.0.0-1252-g44458cce 79957b372307 90352897da85 2024-02-07T12:53:16.017 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (52s) 47s ago 32m 14.9M 2993M 19.0.0-1252-g44458cce 79957b372307 067cd6b3fd2f 2024-02-07T12:53:16.017 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (30s) 25s ago 32m 11.8M 3248M 19.0.0-1252-g44458cce 79957b372307 dbb47a8ad5ac 2024-02-07T12:53:16.017 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (31m) 25s ago 31m 58.0M 3248M 17.2.0 e1d6a67b021e 3bdb22c07537 2024-02-07T12:53:16.017 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (31m) 25s ago 31m 56.7M 3248M 17.2.0 e1d6a67b021e 7859b651e0d9 2024-02-07T12:53:16.017 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (30m) 25s ago 30m 61.9M 3248M 17.2.0 e1d6a67b021e b301e7c75d73 2024-02-07T12:53:16.017 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (7m) 25s ago 30m 67.8M - 2.43.0 a07b618ecd1d c9b607eea1be 2024-02-07T12:53:16.017 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (28m) 47s ago 28m 80.2M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:53:16.017 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (28m) 25s ago 28m 79.9M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:53:16.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:15 smithi172 ceph-mon[67636]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:53:16.239 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:15 smithi019 ceph-mon[88137]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:53:16.240 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:15 smithi019 ceph-mon[82918]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:53:16.420 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:53:16.421 INFO:teuthology.orchestra.run.smithi019.stdout: "mon": { 2024-02-07T12:53:16.421 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 3 2024-02-07T12:53:16.421 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:53:16.421 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr": { 2024-02-07T12:53:16.421 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:53:16.421 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:53:16.421 INFO:teuthology.orchestra.run.smithi019.stdout: "osd": { 2024-02-07T12:53:16.421 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-02-07T12:53:16.421 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 5 2024-02-07T12:53:16.421 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:53:16.421 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw": { 2024-02-07T12:53:16.421 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:53:16.422 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:53:16.422 INFO:teuthology.orchestra.run.smithi019.stdout: "overall": { 2024-02-07T12:53:16.422 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-02-07T12:53:16.422 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 10 2024-02-07T12:53:16.422 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:53:16.422 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:53:16.759 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:53:16.759 INFO:teuthology.orchestra.run.smithi019.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:53:16.759 INFO:teuthology.orchestra.run.smithi019.stdout: "in_progress": true, 2024-02-07T12:53:16.759 INFO:teuthology.orchestra.run.smithi019.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-02-07T12:53:16.759 INFO:teuthology.orchestra.run.smithi019.stdout: "services_complete": [], 2024-02-07T12:53:16.760 INFO:teuthology.orchestra.run.smithi019.stdout: "progress": "5/8 daemons upgraded", 2024-02-07T12:53:16.760 INFO:teuthology.orchestra.run.smithi019.stdout: "message": "Currently upgrading osd daemons", 2024-02-07T12:53:16.760 INFO:teuthology.orchestra.run.smithi019.stdout: "is_paused": false 2024-02-07T12:53:16.760 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:53:16.869 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:16 smithi172 systemd[1]: Stopping Ceph osd.5 for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:53:17.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:16 smithi019 ceph-mon[82918]: from='client.44505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:53:17.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:16 smithi019 ceph-mon[82918]: from='client.44511 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:53:17.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:16 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/3883184761' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:53:17.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:16 smithi019 ceph-mon[88137]: from='client.44505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:53:17.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:16 smithi019 ceph-mon[88137]: from='client.44511 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:53:17.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:16 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/3883184761' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:53:17.208 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:16 smithi172 ceph-mon[67636]: from='client.44505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:53:17.208 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:16 smithi172 ceph-mon[67636]: from='client.44511 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:53:17.208 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:16 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/3883184761' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:53:17.208 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[31771]: 2024-02-07T12:53:17.075+0000 7f02cd0ef700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:53:17.208 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[31771]: 2024-02-07T12:53:17.075+0000 7f02cd0ef700 -1 osd.5 130 *** Got signal Terminated *** 2024-02-07T12:53:17.208 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[31771]: 2024-02-07T12:53:17.075+0000 7f02cd0ef700 -1 osd.5 130 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:53:18.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:17 smithi019 ceph-mon[82918]: from='client.44517 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:53:18.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:17 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:53:18.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:17 smithi019 ceph-mon[82918]: from='client.44529 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:53:18.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:17 smithi019 ceph-mon[82918]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:53:18.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:17 smithi019 ceph-mon[82918]: osd.5 marked itself down and dead 2024-02-07T12:53:18.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:17 smithi019 ceph-mon[88137]: from='client.44517 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:53:18.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:17 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:53:18.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:17 smithi019 ceph-mon[88137]: from='client.44529 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:53:18.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:17 smithi019 ceph-mon[88137]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:53:18.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:17 smithi019 ceph-mon[88137]: osd.5 marked itself down and dead 2024-02-07T12:53:18.362 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:17 smithi172 ceph-mon[67636]: from='client.44517 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:53:18.362 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:17 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:53:18.362 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:17 smithi172 ceph-mon[67636]: from='client.44529 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:53:18.362 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:17 smithi172 ceph-mon[67636]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:53:18.362 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:17 smithi172 ceph-mon[67636]: osd.5 marked itself down and dead 2024-02-07T12:53:18.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:18 smithi172 podman[81886]: 2024-02-07 12:53:18.36084881 +0000 UTC m=+1.415921438 container died 3bdb22c075374a9bcee025b6c58732ac218a304025958513984f4af9160f58f0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5, release=754, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, version=8, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.19.8, RELEASE=HEAD, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=) 2024-02-07T12:53:18.943 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:18 smithi172 podman[81886]: 2024-02-07 12:53:18.742754564 +0000 UTC m=+1.797827194 container cleanup 3bdb22c075374a9bcee025b6c58732ac218a304025958513984f4af9160f58f0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, ceph=True, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , distribution-scope=public, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, name=centos-stream, build-date=2022-05-03T08:36:31.336870, release=754, version=8, architecture=x86_64, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git) 2024-02-07T12:53:18.944 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:18 smithi172 bash[81886]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5 2024-02-07T12:53:19.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:18 smithi019 ceph-mon[82918]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:53:19.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:18 smithi019 ceph-mon[82918]: failed to encode map e131 with expected crc 2024-02-07T12:53:19.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:18 smithi019 ceph-mon[82918]: failed to encode map e131 with expected crc 2024-02-07T12:53:19.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:18 smithi019 ceph-mon[82918]: failed to encode map e131 with expected crc 2024-02-07T12:53:19.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:18 smithi019 ceph-mon[82918]: osdmap e131: 8 total, 7 up, 8 in 2024-02-07T12:53:19.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:18 smithi019 ceph-mon[82918]: failed to encode map e131 with expected crc 2024-02-07T12:53:19.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:18 smithi019 ceph-mon[88137]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:53:19.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:18 smithi019 ceph-mon[88137]: failed to encode map e131 with expected crc 2024-02-07T12:53:19.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:18 smithi019 ceph-mon[88137]: failed to encode map e131 with expected crc 2024-02-07T12:53:19.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:18 smithi019 ceph-mon[88137]: failed to encode map e131 with expected crc 2024-02-07T12:53:19.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:18 smithi019 ceph-mon[88137]: osdmap e131: 8 total, 7 up, 8 in 2024-02-07T12:53:19.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:18 smithi019 ceph-mon[88137]: failed to encode map e131 with expected crc 2024-02-07T12:53:19.357 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:19 smithi172 podman[81914]: 2024-02-07 12:53:19.019355802 +0000 UTC m=+0.654867533 container remove 3bdb22c075374a9bcee025b6c58732ac218a304025958513984f4af9160f58f0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., name=centos-stream, distribution-scope=public, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , ceph=True, vcs-type=git, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, com.redhat.component=centos-stream-container, version=8, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-02-07T12:53:19.358 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:18 smithi172 ceph-mon[67636]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:53:19.358 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:18 smithi172 ceph-mon[67636]: failed to encode map e131 with expected crc 2024-02-07T12:53:19.358 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:18 smithi172 ceph-mon[67636]: failed to encode map e131 with expected crc 2024-02-07T12:53:19.358 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:18 smithi172 ceph-mon[67636]: failed to encode map e131 with expected crc 2024-02-07T12:53:19.358 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:18 smithi172 ceph-mon[67636]: osdmap e131: 8 total, 7 up, 8 in 2024-02-07T12:53:19.358 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:18 smithi172 ceph-mon[67636]: failed to encode map e131 with expected crc 2024-02-07T12:53:19.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:19 smithi172 podman[82006]: 2024-02-07 12:53:19.258872955 +0000 UTC m=+0.018621122 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:53:19.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:19 smithi172 podman[82006]: 2024-02-07 12:53:19.370299304 +0000 UTC m=+0.130047504 container create b1b431561f1e38a022dcc2010ef245d4aba4f05141f2008e306a622d31d01f42 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.2, GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:53:19.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:19 smithi172 podman[82006]: 2024-02-07 12:53:19.581427281 +0000 UTC m=+0.341175440 container init b1b431561f1e38a022dcc2010ef245d4aba4f05141f2008e306a622d31d01f42 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5-deactivate, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:53:19.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:19 smithi172 podman[82006]: 2024-02-07 12:53:19.584722983 +0000 UTC m=+0.344471148 container start b1b431561f1e38a022dcc2010ef245d4aba4f05141f2008e306a622d31d01f42 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:53:19.944 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:19 smithi172 podman[82006]: 2024-02-07 12:53:19.664978567 +0000 UTC m=+0.424726736 container attach b1b431561f1e38a022dcc2010ef245d4aba4f05141f2008e306a622d31d01f42 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240131, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-02-07T12:53:19.944 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:19 smithi172 podman[82006]: 2024-02-07 12:53:19.890209957 +0000 UTC m=+0.649958123 container died b1b431561f1e38a022dcc2010ef245d4aba4f05141f2008e306a622d31d01f42 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5-deactivate, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-02-07T12:53:20.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:19 smithi019 ceph-mon[88137]: pgmap v210: 161 pgs: 10 peering, 16 stale+active+clean, 135 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:53:20.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:19 smithi019 ceph-mon[88137]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2024-02-07T12:53:20.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:19 smithi019 ceph-mon[88137]: failed to encode map e132 with expected crc 2024-02-07T12:53:20.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:19 smithi019 ceph-mon[88137]: failed to encode map e132 with expected crc 2024-02-07T12:53:20.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:19 smithi019 ceph-mon[88137]: osdmap e132: 8 total, 7 up, 8 in 2024-02-07T12:53:20.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:19 smithi019 ceph-mon[88137]: failed to encode map e132 with expected crc 2024-02-07T12:53:20.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:19 smithi019 ceph-mon[88137]: failed to encode map e132 with expected crc 2024-02-07T12:53:20.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:19 smithi019 ceph-mon[88137]: failed to encode map e132 with expected crc 2024-02-07T12:53:20.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:19 smithi019 ceph-mon[82918]: pgmap v210: 161 pgs: 10 peering, 16 stale+active+clean, 135 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:53:20.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:19 smithi019 ceph-mon[82918]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2024-02-07T12:53:20.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:19 smithi019 ceph-mon[82918]: failed to encode map e132 with expected crc 2024-02-07T12:53:20.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:19 smithi019 ceph-mon[82918]: failed to encode map e132 with expected crc 2024-02-07T12:53:20.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:19 smithi019 ceph-mon[82918]: osdmap e132: 8 total, 7 up, 8 in 2024-02-07T12:53:20.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:19 smithi019 ceph-mon[82918]: failed to encode map e132 with expected crc 2024-02-07T12:53:20.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:19 smithi019 ceph-mon[82918]: failed to encode map e132 with expected crc 2024-02-07T12:53:20.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:19 smithi019 ceph-mon[82918]: failed to encode map e132 with expected crc 2024-02-07T12:53:20.236 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:19 smithi172 ceph-mon[67636]: pgmap v210: 161 pgs: 10 peering, 16 stale+active+clean, 135 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:53:20.236 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:19 smithi172 ceph-mon[67636]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2024-02-07T12:53:20.236 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:19 smithi172 ceph-mon[67636]: failed to encode map e132 with expected crc 2024-02-07T12:53:20.237 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:19 smithi172 ceph-mon[67636]: failed to encode map e132 with expected crc 2024-02-07T12:53:20.237 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:19 smithi172 ceph-mon[67636]: osdmap e132: 8 total, 7 up, 8 in 2024-02-07T12:53:20.237 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:19 smithi172 ceph-mon[67636]: failed to encode map e132 with expected crc 2024-02-07T12:53:20.237 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:19 smithi172 ceph-mon[67636]: failed to encode map e132 with expected crc 2024-02-07T12:53:20.237 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:19 smithi172 ceph-mon[67636]: failed to encode map e132 with expected crc 2024-02-07T12:53:20.869 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:20 smithi172 podman[82033]: 2024-02-07 12:53:20.510619634 +0000 UTC m=+0.615460046 container remove b1b431561f1e38a022dcc2010ef245d4aba4f05141f2008e306a622d31d01f42 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5-deactivate, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, io.buildah.version=1.33.2, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240131, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:53:20.869 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:20 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.5.service: Deactivated successfully. 2024-02-07T12:53:20.869 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:20 smithi172 systemd[1]: Stopped Ceph osd.5 for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:53:20.869 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:20 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.5.service: Consumed 15.597s CPU time. 2024-02-07T12:53:20.869 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:20 smithi172 systemd[1]: Starting Ceph osd.5 for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:53:21.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:21 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:21.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:21 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:53:21.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:21 smithi172 ceph-mon[67636]: pgmap v212: 161 pgs: 3 active+undersized+degraded, 10 peering, 12 stale+active+clean, 6 active+undersized, 130 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 14/627 objects degraded (2.233%) 2024-02-07T12:53:21.225 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:20 smithi172 podman[82139]: 2024-02-07 12:53:20.850078611 +0000 UTC m=+0.017898092 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:53:21.225 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:20 smithi172 podman[82139]: 2024-02-07 12:53:20.995094785 +0000 UTC m=+0.162914261 container create 1e0942ba7d7cdd6431b4b6d99884a0e461b0fb2be5e329405dc8649392ea3d66 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5-activate, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:53:21.225 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:21 smithi172 podman[82139]: 2024-02-07 12:53:21.223484871 +0000 UTC m=+0.391304356 container init 1e0942ba7d7cdd6431b4b6d99884a0e461b0fb2be5e329405dc8649392ea3d66 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5-activate, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-02-07T12:53:21.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:21 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:21.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:21 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:53:21.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:21 smithi019 ceph-mon[88137]: pgmap v212: 161 pgs: 3 active+undersized+degraded, 10 peering, 12 stale+active+clean, 6 active+undersized, 130 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 14/627 objects degraded (2.233%) 2024-02-07T12:53:21.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:21 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:21.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:21 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:53:21.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:21 smithi019 ceph-mon[82918]: pgmap v212: 161 pgs: 3 active+undersized+degraded, 10 peering, 12 stale+active+clean, 6 active+undersized, 130 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 14/627 objects degraded (2.233%) 2024-02-07T12:53:21.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:21 smithi172 podman[82139]: 2024-02-07 12:53:21.226914708 +0000 UTC m=+0.394734183 container start 1e0942ba7d7cdd6431b4b6d99884a0e461b0fb2be5e329405dc8649392ea3d66 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5-activate, ceph=True, org.label-schema.build-date=20240131, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-02-07T12:53:21.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:21 smithi172 podman[82139]: 2024-02-07 12:53:21.298395655 +0000 UTC m=+0.466215147 container attach 1e0942ba7d7cdd6431b4b6d99884a0e461b0fb2be5e329405dc8649392ea3d66 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5-activate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240131, io.buildah.version=1.33.2, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:53:22.003 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:21 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5-activate[82154]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-02-07T12:53:22.003 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:21 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5-activate[82154]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-02-07T12:53:22.003 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:21 smithi172 bash[82139]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-02-07T12:53:22.003 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:21 smithi172 bash[82139]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-02-07T12:53:22.004 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:21 smithi172 bash[82139]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-02-07T12:53:22.004 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:21 smithi172 bash[82139]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-02-07T12:53:22.004 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:21 smithi172 bash[82139]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-02-07T12:53:22.004 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:21 smithi172 bash[82139]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-02-07T12:53:22.004 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:21 smithi172 bash[82139]: --> ceph-volume raw activate successful for osd ID: 5 2024-02-07T12:53:22.004 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:21 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5-activate[82154]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-02-07T12:53:22.004 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:21 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5-activate[82154]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-02-07T12:53:22.004 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:21 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5-activate[82154]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-02-07T12:53:22.004 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:21 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5-activate[82154]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-02-07T12:53:22.004 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:21 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5-activate[82154]: --> ceph-volume raw activate successful for osd ID: 5 2024-02-07T12:53:22.004 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:21 smithi172 podman[82139]: 2024-02-07 12:53:21.692896291 +0000 UTC m=+0.860715764 container died 1e0942ba7d7cdd6431b4b6d99884a0e461b0fb2be5e329405dc8649392ea3d66 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5-activate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240131, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.2, org.label-schema.vendor=CentOS) 2024-02-07T12:53:22.312 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:22 smithi172 ceph-mon[67636]: Health check failed: Degraded data redundancy: 14/627 objects degraded (2.233%), 3 pgs degraded (PG_DEGRADED) 2024-02-07T12:53:22.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:22 smithi019 ceph-mon[82918]: Health check failed: Degraded data redundancy: 14/627 objects degraded (2.233%), 3 pgs degraded (PG_DEGRADED) 2024-02-07T12:53:22.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:22 smithi019 ceph-mon[88137]: Health check failed: Degraded data redundancy: 14/627 objects degraded (2.233%), 3 pgs degraded (PG_DEGRADED) 2024-02-07T12:53:22.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:22 smithi172 podman[82281]: 2024-02-07 12:53:22.310841973 +0000 UTC m=+0.612843966 container remove 1e0942ba7d7cdd6431b4b6d99884a0e461b0fb2be5e329405dc8649392ea3d66 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5-activate, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240131, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2) 2024-02-07T12:53:22.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:22 smithi172 podman[82347]: 2024-02-07 12:53:22.435854186 +0000 UTC m=+0.018403006 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:53:22.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:22 smithi172 podman[82347]: 2024-02-07 12:53:22.55331249 +0000 UTC m=+0.135861314 container create d139d35fec5787bb892e1f7524804b305300c589c4af4378cb4ed77ff809e13c (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5, org.label-schema.build-date=20240131, io.buildah.version=1.33.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-02-07T12:53:23.119 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:22 smithi172 podman[82347]: 2024-02-07 12:53:22.730718879 +0000 UTC m=+0.313267697 container init d139d35fec5787bb892e1f7524804b305300c589c4af4378cb4ed77ff809e13c (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240131, io.buildah.version=1.33.2) 2024-02-07T12:53:23.119 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:22 smithi172 podman[82347]: 2024-02-07 12:53:22.734025418 +0000 UTC m=+0.316574237 container start d139d35fec5787bb892e1f7524804b305300c589c4af4378cb4ed77ff809e13c (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:53:23.119 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:22 smithi172 ceph-osd[82366]: -- 172.21.15.172:0/2305707936 <== mon.2 v2:172.21.15.172:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55899db376c0 con 0x55899db09400 2024-02-07T12:53:23.119 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:22 smithi172 bash[82347]: d139d35fec5787bb892e1f7524804b305300c589c4af4378cb4ed77ff809e13c 2024-02-07T12:53:23.119 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:22 smithi172 systemd[1]: Started Ceph osd.5 for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:53:23.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:23 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:53:23.275+0000 7f26e89e57c0 -1 Falling back to public interface 2024-02-07T12:53:24.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:23 smithi172 ceph-mon[67636]: pgmap v213: 161 pgs: 14 active+undersized+degraded, 10 peering, 5 stale+active+clean, 23 active+undersized, 109 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 42/627 objects degraded (6.699%) 2024-02-07T12:53:24.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:23 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:24.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:23 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:24.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:23 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:53:24.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:53:23] "GET /metrics HTTP/1.1" 200 38265 "" "Prometheus/2.43.0" 2024-02-07T12:53:24.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:23 smithi019 ceph-mon[82918]: pgmap v213: 161 pgs: 14 active+undersized+degraded, 10 peering, 5 stale+active+clean, 23 active+undersized, 109 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 42/627 objects degraded (6.699%) 2024-02-07T12:53:24.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:23 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:24.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:23 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:24.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:23 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:53:24.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:23 smithi019 ceph-mon[88137]: pgmap v213: 161 pgs: 14 active+undersized+degraded, 10 peering, 5 stale+active+clean, 23 active+undersized, 109 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 42/627 objects degraded (6.699%) 2024-02-07T12:53:24.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:23 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:24.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:23 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:24.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:23 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:53:25.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:24 smithi019 ceph-mon[82918]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2024-02-07T12:53:25.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:24 smithi019 ceph-mon[88137]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2024-02-07T12:53:25.357 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:24 smithi172 ceph-mon[67636]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2024-02-07T12:53:26.237 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:25 smithi172 ceph-mon[67636]: pgmap v214: 161 pgs: 22 active+undersized+degraded, 37 active+undersized, 102 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 74/627 objects degraded (11.802%) 2024-02-07T12:53:26.237 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:53:25.985+0000 7f26e89e57c0 -1 osd.5 0 read_superblock omap replica is missing. 2024-02-07T12:53:26.237 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:26 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:53:26.010+0000 7f26e89e57c0 -1 osd.5 130 log_to_monitors true 2024-02-07T12:53:26.250 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:25 smithi019 ceph-mon[82918]: pgmap v214: 161 pgs: 22 active+undersized+degraded, 37 active+undersized, 102 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 74/627 objects degraded (11.802%) 2024-02-07T12:53:26.250 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:25 smithi019 ceph-mon[88137]: pgmap v214: 161 pgs: 22 active+undersized+degraded, 37 active+undersized, 102 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 74/627 objects degraded (11.802%) 2024-02-07T12:53:27.099 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:26 smithi172 ceph-mon[67636]: from='osd.5 [v2:172.21.15.172:6808/2819370796,v1:172.21.15.172:6809/2819370796]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-07T12:53:27.100 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:26 smithi172 ceph-mon[67636]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-07T12:53:27.100 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:53:26 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:53:26.956+0000 7f26d9e44700 -1 osd.5 130 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:53:27.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:26 smithi019 ceph-mon[88137]: from='osd.5 [v2:172.21.15.172:6808/2819370796,v1:172.21.15.172:6809/2819370796]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-07T12:53:27.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:26 smithi019 ceph-mon[88137]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-07T12:53:27.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:26 smithi019 ceph-mon[82918]: from='osd.5 [v2:172.21.15.172:6808/2819370796,v1:172.21.15.172:6809/2819370796]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-07T12:53:27.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:26 smithi019 ceph-mon[82918]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-07T12:53:28.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:27 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:53:28.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:27 smithi019 ceph-mon[88137]: pgmap v215: 161 pgs: 22 active+undersized+degraded, 37 active+undersized, 102 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 74/627 objects degraded (11.802%) 2024-02-07T12:53:28.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:27 smithi019 ceph-mon[88137]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-02-07T12:53:28.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:27 smithi019 ceph-mon[88137]: osdmap e133: 8 total, 7 up, 8 in 2024-02-07T12:53:28.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:27 smithi019 ceph-mon[88137]: from='osd.5 [v2:172.21.15.172:6808/2819370796,v1:172.21.15.172:6809/2819370796]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:53:28.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:27 smithi019 ceph-mon[88137]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:53:28.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:27 smithi019 ceph-mon[88137]: failed to encode map e133 with expected crc 2024-02-07T12:53:28.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:27 smithi019 ceph-mon[88137]: failed to encode map e133 with expected crc 2024-02-07T12:53:28.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:27 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:28.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:27 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:28.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:27 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:53:28.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:27 smithi019 ceph-mon[82918]: pgmap v215: 161 pgs: 22 active+undersized+degraded, 37 active+undersized, 102 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 74/627 objects degraded (11.802%) 2024-02-07T12:53:28.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:27 smithi019 ceph-mon[82918]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-02-07T12:53:28.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:27 smithi019 ceph-mon[82918]: osdmap e133: 8 total, 7 up, 8 in 2024-02-07T12:53:28.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:27 smithi019 ceph-mon[82918]: from='osd.5 [v2:172.21.15.172:6808/2819370796,v1:172.21.15.172:6809/2819370796]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:53:28.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:27 smithi019 ceph-mon[82918]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:53:28.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:27 smithi019 ceph-mon[82918]: failed to encode map e133 with expected crc 2024-02-07T12:53:28.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:27 smithi019 ceph-mon[82918]: failed to encode map e133 with expected crc 2024-02-07T12:53:28.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:27 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:28.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:27 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:28.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:27 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:53:28.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:27 smithi172 ceph-mon[67636]: pgmap v215: 161 pgs: 22 active+undersized+degraded, 37 active+undersized, 102 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 74/627 objects degraded (11.802%) 2024-02-07T12:53:28.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:27 smithi172 ceph-mon[67636]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-02-07T12:53:28.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:27 smithi172 ceph-mon[67636]: osdmap e133: 8 total, 7 up, 8 in 2024-02-07T12:53:28.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:27 smithi172 ceph-mon[67636]: from='osd.5 [v2:172.21.15.172:6808/2819370796,v1:172.21.15.172:6809/2819370796]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:53:28.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:27 smithi172 ceph-mon[67636]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:53:28.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:27 smithi172 ceph-mon[67636]: failed to encode map e133 with expected crc 2024-02-07T12:53:28.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:27 smithi172 ceph-mon[67636]: failed to encode map e133 with expected crc 2024-02-07T12:53:28.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:27 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:28.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:27 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:29.244 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:28 smithi172 ceph-mon[67636]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:53:29.244 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:28 smithi172 ceph-mon[67636]: failed to encode map e134 with expected crc 2024-02-07T12:53:29.244 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:28 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:53:29.244 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:28 smithi172 ceph-mon[67636]: failed to encode map e134 with expected crc 2024-02-07T12:53:29.244 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:28 smithi172 ceph-mon[67636]: failed to encode map e134 with expected crc 2024-02-07T12:53:29.244 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:28 smithi172 ceph-mon[67636]: osd.5 [v2:172.21.15.172:6808/2819370796,v1:172.21.15.172:6809/2819370796] boot 2024-02-07T12:53:29.244 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:28 smithi172 ceph-mon[67636]: osdmap e134: 8 total, 8 up, 8 in 2024-02-07T12:53:29.244 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:28 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:29.245 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:28 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:29.245 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:28 smithi172 ceph-mon[67636]: osdmap e135: 8 total, 8 up, 8 in 2024-02-07T12:53:29.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:28 smithi019 ceph-mon[88137]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:53:29.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:28 smithi019 ceph-mon[88137]: failed to encode map e134 with expected crc 2024-02-07T12:53:29.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:28 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:53:29.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:28 smithi019 ceph-mon[88137]: failed to encode map e134 with expected crc 2024-02-07T12:53:29.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:28 smithi019 ceph-mon[88137]: failed to encode map e134 with expected crc 2024-02-07T12:53:29.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:28 smithi019 ceph-mon[88137]: osd.5 [v2:172.21.15.172:6808/2819370796,v1:172.21.15.172:6809/2819370796] boot 2024-02-07T12:53:29.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:28 smithi019 ceph-mon[88137]: osdmap e134: 8 total, 8 up, 8 in 2024-02-07T12:53:29.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:28 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:29.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:28 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:29.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:28 smithi019 ceph-mon[88137]: osdmap e135: 8 total, 8 up, 8 in 2024-02-07T12:53:29.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:28 smithi019 ceph-mon[82918]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:53:29.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:28 smithi019 ceph-mon[82918]: failed to encode map e134 with expected crc 2024-02-07T12:53:29.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:28 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:53:29.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:28 smithi019 ceph-mon[82918]: failed to encode map e134 with expected crc 2024-02-07T12:53:29.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:28 smithi019 ceph-mon[82918]: failed to encode map e134 with expected crc 2024-02-07T12:53:29.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:28 smithi019 ceph-mon[82918]: osd.5 [v2:172.21.15.172:6808/2819370796,v1:172.21.15.172:6809/2819370796] boot 2024-02-07T12:53:29.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:28 smithi019 ceph-mon[82918]: osdmap e134: 8 total, 8 up, 8 in 2024-02-07T12:53:29.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:28 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:29.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:28 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:29.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:28 smithi019 ceph-mon[82918]: osdmap e135: 8 total, 8 up, 8 in 2024-02-07T12:53:30.242 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:29 smithi172 ceph-mon[67636]: pgmap v218: 161 pgs: 5 peering, 20 active+undersized+degraded, 34 active+undersized, 102 active+clean; 585 KiB data, 208 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-02-07T12:53:30.242 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:29 smithi172 ceph-mon[67636]: failed to encode map e135 with expected crc 2024-02-07T12:53:30.242 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:29 smithi172 ceph-mon[67636]: failed to encode map e135 with expected crc 2024-02-07T12:53:30.242 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:29 smithi172 ceph-mon[67636]: failed to encode map e135 with expected crc 2024-02-07T12:53:30.242 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:29 smithi172 ceph-mon[67636]: failed to encode map e135 with expected crc 2024-02-07T12:53:30.243 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:29 smithi172 ceph-mon[67636]: failed to encode map e135 with expected crc 2024-02-07T12:53:30.243 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:29 smithi172 ceph-mon[67636]: Health check update: Degraded data redundancy: 68/627 objects degraded (10.845%), 20 pgs degraded (PG_DEGRADED) 2024-02-07T12:53:30.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:29 smithi019 ceph-mon[82918]: pgmap v218: 161 pgs: 5 peering, 20 active+undersized+degraded, 34 active+undersized, 102 active+clean; 585 KiB data, 208 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-02-07T12:53:30.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:29 smithi019 ceph-mon[82918]: failed to encode map e135 with expected crc 2024-02-07T12:53:30.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:29 smithi019 ceph-mon[82918]: failed to encode map e135 with expected crc 2024-02-07T12:53:30.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:29 smithi019 ceph-mon[82918]: failed to encode map e135 with expected crc 2024-02-07T12:53:30.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:29 smithi019 ceph-mon[82918]: failed to encode map e135 with expected crc 2024-02-07T12:53:30.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:29 smithi019 ceph-mon[82918]: failed to encode map e135 with expected crc 2024-02-07T12:53:30.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:29 smithi019 ceph-mon[82918]: Health check update: Degraded data redundancy: 68/627 objects degraded (10.845%), 20 pgs degraded (PG_DEGRADED) 2024-02-07T12:53:30.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:29 smithi019 ceph-mon[88137]: pgmap v218: 161 pgs: 5 peering, 20 active+undersized+degraded, 34 active+undersized, 102 active+clean; 585 KiB data, 208 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-02-07T12:53:30.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:29 smithi019 ceph-mon[88137]: failed to encode map e135 with expected crc 2024-02-07T12:53:30.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:29 smithi019 ceph-mon[88137]: failed to encode map e135 with expected crc 2024-02-07T12:53:30.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:29 smithi019 ceph-mon[88137]: failed to encode map e135 with expected crc 2024-02-07T12:53:30.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:29 smithi019 ceph-mon[88137]: failed to encode map e135 with expected crc 2024-02-07T12:53:30.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:29 smithi019 ceph-mon[88137]: failed to encode map e135 with expected crc 2024-02-07T12:53:30.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:29 smithi019 ceph-mon[88137]: Health check update: Degraded data redundancy: 68/627 objects degraded (10.845%), 20 pgs degraded (PG_DEGRADED) 2024-02-07T12:53:32.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:31 smithi172 ceph-mon[67636]: pgmap v220: 161 pgs: 5 peering, 19 active+undersized+degraded, 30 active+undersized, 107 active+clean; 585 KiB data, 208 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s; 67/627 objects degraded (10.686%) 2024-02-07T12:53:32.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:31 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:32.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:31 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:32.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:31 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:53:32.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:31 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:53:32.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:31 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:32.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:31 smithi019 ceph-mon[88137]: pgmap v220: 161 pgs: 5 peering, 19 active+undersized+degraded, 30 active+undersized, 107 active+clean; 585 KiB data, 208 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s; 67/627 objects degraded (10.686%) 2024-02-07T12:53:32.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:31 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:32.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:31 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:32.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:31 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:53:32.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:31 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:53:32.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:31 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:32.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:31 smithi019 ceph-mon[82918]: pgmap v220: 161 pgs: 5 peering, 19 active+undersized+degraded, 30 active+undersized, 107 active+clean; 585 KiB data, 208 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s; 67/627 objects degraded (10.686%) 2024-02-07T12:53:32.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:31 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:32.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:31 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:32.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:31 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:53:32.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:31 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:53:32.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:31 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:33.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:32 smithi019 ceph-mon[82918]: pgmap v221: 161 pgs: 5 peering, 11 active+undersized+degraded, 24 active+undersized, 121 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1017 B/s rd, 0 op/s; 45/627 objects degraded (7.177%) 2024-02-07T12:53:33.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:32 smithi019 ceph-mon[82918]: pgmap v222: 161 pgs: 5 peering, 11 active+undersized+degraded, 24 active+undersized, 121 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 45/627 objects degraded (7.177%) 2024-02-07T12:53:33.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:32 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:53:33.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:32 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:53:33.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:32 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:53:33.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:32 smithi019 ceph-mon[88137]: pgmap v221: 161 pgs: 5 peering, 11 active+undersized+degraded, 24 active+undersized, 121 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1017 B/s rd, 0 op/s; 45/627 objects degraded (7.177%) 2024-02-07T12:53:33.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:32 smithi019 ceph-mon[88137]: pgmap v222: 161 pgs: 5 peering, 11 active+undersized+degraded, 24 active+undersized, 121 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 45/627 objects degraded (7.177%) 2024-02-07T12:53:33.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:32 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:53:33.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:32 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:53:33.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:32 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:53:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:32 smithi172 ceph-mon[67636]: pgmap v221: 161 pgs: 5 peering, 11 active+undersized+degraded, 24 active+undersized, 121 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1017 B/s rd, 0 op/s; 45/627 objects degraded (7.177%) 2024-02-07T12:53:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:32 smithi172 ceph-mon[67636]: pgmap v222: 161 pgs: 5 peering, 11 active+undersized+degraded, 24 active+undersized, 121 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 45/627 objects degraded (7.177%) 2024-02-07T12:53:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:32 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:53:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:32 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:53:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:32 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:53:33.844 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:53:33.700+0000 7fb71d671700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:53:33.844 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:53:33.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:53:33.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:53:33.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:53:33.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:53:33.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:53:33.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:53:33.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:53:33.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:53:33.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:53:33.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:53:33.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:53:33.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:53:33.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:53:33.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:53:33.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:53:33.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:53:33.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:53:33.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:53:33.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:53:33.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:53:33.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:53:33.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:53:33.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:53:33.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:53:33.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:53:33.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:53:33.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:53:33.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:53:33.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:53:33.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:53:33.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:53:33.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:53:33.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:53:33.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:53:33.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:53:33.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:53:33.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:53:33.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:53:33.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:53:33.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:53:33.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:53:33.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:53:33.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:53:33.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:53:33.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:53:33.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:53:33.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:53:33.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:53:33.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:53:33.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:53:33.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:53:33.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:53:33.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:53:33.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:53:33.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:53:33.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:53:33.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:53:33.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:53:33.715+0000 7fb74ced9700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-02-07T12:53:34.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:53:33] "GET /metrics HTTP/1.1" 200 38265 "" "Prometheus/2.43.0" 2024-02-07T12:53:34.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-mon[88137]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:53:34.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-mon[88137]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:53:34.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:53:34.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:53:34.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:53:34.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:53:34.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-07T12:53:34.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-mon[82918]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:53:34.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-mon[82918]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:53:34.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:53:34.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:53:34.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:53:34.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:53:34.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:33 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-07T12:53:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:33 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:53:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:33 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:53:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:33 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:53:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:33 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:53:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:33 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:53:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:33 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:53:34.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:33 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-07T12:53:35.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:34 smithi019 ceph-mon[82918]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:53:35.344 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:53:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:53:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:53:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:53:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:53:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:53:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:53:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:53:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:53:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:53:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:53:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:53:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:53:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:53:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:53:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:53:35.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:53:35.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:53:35.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:53:35.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:53:35.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:53:35.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:53:35.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:53:35.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:53:35.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:53:35.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:53:35.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:53:35.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:53:35.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:53:35.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:53:35.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:53:35.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:53:35.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:53:35.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:53:35.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:53:35.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:53:35.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:53:35.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:53:35.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:53:35.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:53:35.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:53:35.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:53:35.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:53:35.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:53:35.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:53:35.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:53:35.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:53:35.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:53:35.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:53:35.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:53:35.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:53:35.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:53:35.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:53:35.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:34 smithi019 ceph-mon[88137]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:53:35.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:53:35.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:53:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:53:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:53:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:53:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:53:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:53:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:53:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:53:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:53:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:53:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:53:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:53:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:53:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:53:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:53:35.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:53:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:53:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:53:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:53:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:53:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:53:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:53:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:53:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:53:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:53:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:53:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:53:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:53:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:53:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:53:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:53:35.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:53:35.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:53:35.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:53:35.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:53:35.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:53:35.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:53:35.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:53:35.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:53:35.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:53:35.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:53:35.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:53:35.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:53:35.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:53:35.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:53:35.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:53:35.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:53:35.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:53:35.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:53:35.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:53:35.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:53:35.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:53:35.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:53:35.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:53:35.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:53:35.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:53:35.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:53:35.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:34 smithi019 ceph-mon[82918]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-07T12:53:35.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:34 smithi019 ceph-mon[82918]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-02-07T12:53:35.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:34 smithi019 ceph-mon[82918]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-02-07T12:53:35.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:34 smithi019 ceph-mon[82918]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/627 objects degraded (7.177%), 11 pgs degraded) 2024-02-07T12:53:35.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:53:35.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:53:35.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:53:35.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:53:35.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:53:35.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:34 smithi019 ceph-mon[88137]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-07T12:53:35.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:34 smithi019 ceph-mon[88137]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-02-07T12:53:35.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:34 smithi019 ceph-mon[88137]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-02-07T12:53:35.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:34 smithi019 ceph-mon[88137]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/627 objects degraded (7.177%), 11 pgs degraded) 2024-02-07T12:53:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:34 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:53:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:53:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:53:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:53:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:53:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:53:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:53:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:53:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:53:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:53:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:53:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:53:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:53:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:53:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:53:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:53:35.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:53:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:53:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:53:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:53:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:53:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:53:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:53:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:53:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:53:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:53:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:53:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:53:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:53:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:53:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:53:35.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:53:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:53:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:53:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:53:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:53:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:53:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:53:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:53:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:53:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:53:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:53:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:53:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:53:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:53:35.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:53:35.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:53:35.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:53:35.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:53:35.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:53:35.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:53:35.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:53:35.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:53:35.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:53:35.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:53:35.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:53:35.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:53:35.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:53:35.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:53:35.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:34 smithi172 ceph-mon[67636]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-07T12:53:35.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:34 smithi172 ceph-mon[67636]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-02-07T12:53:35.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:34 smithi172 ceph-mon[67636]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-02-07T12:53:35.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:34 smithi172 ceph-mon[67636]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/627 objects degraded (7.177%), 11 pgs degraded) 2024-02-07T12:53:36.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:36 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:36.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:36 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:53:36.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:36 smithi019 ceph-mon[88137]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 728 B/s rd, 0 op/s 2024-02-07T12:53:36.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:36 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:36.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:36 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:53:36.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:36 smithi019 ceph-mon[82918]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 728 B/s rd, 0 op/s 2024-02-07T12:53:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:36 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:36 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:53:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:36 smithi172 ceph-mon[67636]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 728 B/s rd, 0 op/s 2024-02-07T12:53:37.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:37 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:53:37.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:37 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:53:37.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:37 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:53:38.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:38 smithi172 ceph-mon[67636]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 571 B/s rd, 0 op/s 2024-02-07T12:53:38.432 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:38 smithi019 ceph-mon[88137]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 571 B/s rd, 0 op/s 2024-02-07T12:53:38.432 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:38 smithi019 ceph-mon[82918]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 571 B/s rd, 0 op/s 2024-02-07T12:53:41.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:40 smithi019 ceph-mon[88137]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1020 B/s rd, 0 op/s 2024-02-07T12:53:41.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:41.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:40 smithi019 ceph-mon[82918]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1020 B/s rd, 0 op/s 2024-02-07T12:53:41.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:41.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:40 smithi172 ceph-mon[67636]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1020 B/s rd, 0 op/s 2024-02-07T12:53:41.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:43.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:42 smithi019 ceph-mon[88137]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-02-07T12:53:43.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:42 smithi019 ceph-mon[82918]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-02-07T12:53:43.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:42 smithi172 ceph-mon[67636]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-02-07T12:53:44.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:53:43] "GET /metrics HTTP/1.1" 200 38274 "" "Prometheus/2.43.0" 2024-02-07T12:53:45.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:44 smithi019 ceph-mon[82918]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:53:45.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:44 smithi019 ceph-mon[88137]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:53:45.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:44 smithi172 ceph-mon[67636]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:53:47.132 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:53:47.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:46 smithi019 ceph-mon[88137]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-02-07T12:53:47.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:46 smithi019 ceph-mon[82918]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-02-07T12:53:47.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:46 smithi172 ceph-mon[67636]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-02-07T12:53:47.794 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:53:47.794 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (8m) 79s ago 29m 27.3M - 0.25.0 c8568f914cd2 fec9255f35e7 2024-02-07T12:53:47.794 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (7m) 20s ago 29m 84.7M - 9.4.12 a0ea5e3c46d9 aaffac63bc28 2024-02-07T12:53:47.794 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 unknown 79s ago 29m 63.5M - 3.5 e1d6a67b021e 5113aa77e3f2 2024-02-07T12:53:47.794 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283,8765 running (9m) 20s ago 34m 451M - 19.0.0-1252-g44458cce 79957b372307 5aa0c04fdfd7 2024-02-07T12:53:47.794 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (19m) 79s ago 36m 509M - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:53:47.794 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (5m) 79s ago 36m 53.6M 2048M 19.0.0-1252-g44458cce 79957b372307 8b1b0e21b1c5 2024-02-07T12:53:47.794 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (6m) 20s ago 34m 52.2M 2048M 19.0.0-1252-g44458cce 79957b372307 00f454bbac01 2024-02-07T12:53:47.795 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (5m) 79s ago 34m 43.3M 2048M 19.0.0-1252-g44458cce 79957b372307 6214003f6e08 2024-02-07T12:53:47.795 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (8m) 79s ago 30m 16.0M - 1.5.0 0da6a335fe13 c4bdcb1c4d49 2024-02-07T12:53:47.795 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (8m) 20s ago 30m 16.8M - 1.5.0 0da6a335fe13 04ea81f79d85 2024-02-07T12:53:47.795 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (3m) 79s ago 34m 56.0M 2993M 19.0.0-1252-g44458cce 79957b372307 78eddd34a903 2024-02-07T12:53:47.795 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (3m) 79s ago 33m 44.0M 2993M 19.0.0-1252-g44458cce 79957b372307 c37447e76c38 2024-02-07T12:53:47.795 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (2m) 79s ago 33m 50.2M 2993M 19.0.0-1252-g44458cce 79957b372307 90352897da85 2024-02-07T12:53:47.795 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (84s) 79s ago 32m 14.9M 2993M 19.0.0-1252-g44458cce 79957b372307 067cd6b3fd2f 2024-02-07T12:53:47.795 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (62s) 20s ago 32m 43.1M 3248M 19.0.0-1252-g44458cce 79957b372307 dbb47a8ad5ac 2024-02-07T12:53:47.795 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (25s) 20s ago 32m 12.6M 3248M 19.0.0-1252-g44458cce 79957b372307 d139d35fec57 2024-02-07T12:53:47.795 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (31m) 20s ago 31m 57.5M 3248M 17.2.0 e1d6a67b021e 7859b651e0d9 2024-02-07T12:53:47.795 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (31m) 20s ago 31m 63.0M 3248M 17.2.0 e1d6a67b021e b301e7c75d73 2024-02-07T12:53:47.795 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (8m) 20s ago 30m 68.0M - 2.43.0 a07b618ecd1d c9b607eea1be 2024-02-07T12:53:47.795 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (29m) 79s ago 29m 80.2M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:53:47.795 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (29m) 20s ago 29m 80.1M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:53:48.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:47 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:53:48.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:47 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:53:48.208 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:53:48.208 INFO:teuthology.orchestra.run.smithi019.stdout: "mon": { 2024-02-07T12:53:48.208 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 3 2024-02-07T12:53:48.208 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:53:48.208 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr": { 2024-02-07T12:53:48.208 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:53:48.208 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:53:48.208 INFO:teuthology.orchestra.run.smithi019.stdout: "osd": { 2024-02-07T12:53:48.208 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-02-07T12:53:48.208 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 6 2024-02-07T12:53:48.208 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:53:48.208 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw": { 2024-02-07T12:53:48.209 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:53:48.209 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:53:48.209 INFO:teuthology.orchestra.run.smithi019.stdout: "overall": { 2024-02-07T12:53:48.209 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-02-07T12:53:48.209 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 11 2024-02-07T12:53:48.209 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:53:48.209 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:53:48.270 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:47 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:53:48.566 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:53:48.566 INFO:teuthology.orchestra.run.smithi019.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:53:48.566 INFO:teuthology.orchestra.run.smithi019.stdout: "in_progress": true, 2024-02-07T12:53:48.566 INFO:teuthology.orchestra.run.smithi019.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-02-07T12:53:48.567 INFO:teuthology.orchestra.run.smithi019.stdout: "services_complete": [], 2024-02-07T12:53:48.567 INFO:teuthology.orchestra.run.smithi019.stdout: "progress": "6/8 daemons upgraded", 2024-02-07T12:53:48.567 INFO:teuthology.orchestra.run.smithi019.stdout: "message": "Currently upgrading osd daemons", 2024-02-07T12:53:48.567 INFO:teuthology.orchestra.run.smithi019.stdout: "is_paused": false 2024-02-07T12:53:48.567 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:53:49.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:48 smithi172 ceph-mon[67636]: from='client.54478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:53:49.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:48 smithi172 ceph-mon[67636]: from='client.44544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:53:49.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:48 smithi172 ceph-mon[67636]: from='client.44550 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:53:49.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:48 smithi172 ceph-mon[67636]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-02-07T12:53:49.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:48 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/2887601221' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:53:49.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-07T12:53:49.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:48 smithi019 ceph-mon[88137]: from='client.54478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:53:49.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:48 smithi019 ceph-mon[88137]: from='client.44544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:53:49.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:48 smithi019 ceph-mon[88137]: from='client.44550 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:53:49.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:48 smithi019 ceph-mon[88137]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-02-07T12:53:49.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:48 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/2887601221' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:53:49.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-07T12:53:49.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:48 smithi019 ceph-mon[82918]: from='client.54478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:53:49.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:48 smithi019 ceph-mon[82918]: from='client.44544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:53:49.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:48 smithi019 ceph-mon[82918]: from='client.44550 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:53:49.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:48 smithi019 ceph-mon[82918]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-02-07T12:53:49.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:48 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/2887601221' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:53:49.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-07T12:53:50.272 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:49 smithi172 ceph-mon[67636]: from='client.44562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:53:50.272 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:49 smithi172 ceph-mon[67636]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-07T12:53:50.272 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:49 smithi172 ceph-mon[67636]: Upgrade: osd.6 is safe to restart 2024-02-07T12:53:50.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:49 smithi019 ceph-mon[88137]: from='client.44562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:53:50.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:49 smithi019 ceph-mon[88137]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-07T12:53:50.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:49 smithi019 ceph-mon[88137]: Upgrade: osd.6 is safe to restart 2024-02-07T12:53:50.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:49 smithi019 ceph-mon[82918]: from='client.44562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:53:50.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:49 smithi019 ceph-mon[82918]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-07T12:53:50.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:49 smithi019 ceph-mon[82918]: Upgrade: osd.6 is safe to restart 2024-02-07T12:53:51.183 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:50 smithi172 ceph-mon[67636]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:53:51.183 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:50 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:53:51.183 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:50 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:51.183 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:50 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-02-07T12:53:51.183 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:50 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:53:51.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:50 smithi019 ceph-mon[82918]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:53:51.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:50 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:53:51.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:50 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:51.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:50 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-02-07T12:53:51.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:50 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:53:51.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:50 smithi019 ceph-mon[88137]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:53:51.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:50 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:53:51.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:50 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:53:51.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:50 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-02-07T12:53:51.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:50 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:53:52.229 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:51 smithi172 ceph-mon[67636]: Upgrade: Updating osd.6 2024-02-07T12:53:52.229 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:51 smithi172 ceph-mon[67636]: Deploying daemon osd.6 on smithi172 2024-02-07T12:53:52.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:51 smithi019 ceph-mon[82918]: Upgrade: Updating osd.6 2024-02-07T12:53:52.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:51 smithi019 ceph-mon[82918]: Deploying daemon osd.6 on smithi172 2024-02-07T12:53:52.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:51 smithi019 ceph-mon[88137]: Upgrade: Updating osd.6 2024-02-07T12:53:52.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:51 smithi019 ceph-mon[88137]: Deploying daemon osd.6 on smithi172 2024-02-07T12:53:53.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:52 smithi019 ceph-mon[82918]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:53:53.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:52 smithi019 ceph-mon[88137]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:53:53.352 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:52 smithi172 ceph-mon[67636]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:53:53.975 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:53 smithi172 systemd[1]: Stopping Ceph osd.6 for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:53:54.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:53 smithi019 ceph-mon[82918]: osd.6 marked itself down and dead 2024-02-07T12:53:54.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:53 smithi019 ceph-mon[88137]: osd.6 marked itself down and dead 2024-02-07T12:53:54.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:53:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:53:53] "GET /metrics HTTP/1.1" 200 38275 "" "Prometheus/2.43.0" 2024-02-07T12:53:54.363 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:53 smithi172 ceph-mon[67636]: osd.6 marked itself down and dead 2024-02-07T12:53:54.364 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:53 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[34650]: 2024-02-07T12:53:53.971+0000 7f55f7ee4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:53:54.364 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:53 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[34650]: 2024-02-07T12:53:53.971+0000 7f55f7ee4700 -1 osd.6 135 *** Got signal Terminated *** 2024-02-07T12:53:54.364 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:53 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[34650]: 2024-02-07T12:53:53.971+0000 7f55f7ee4700 -1 osd.6 135 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:53:54.619 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:54 smithi172 podman[86442]: 2024-02-07 12:53:54.361739557 +0000 UTC m=+0.507109248 container died 7859b651e0d905ed705c599d03809e1f15c2621ab7d12be145131dff38a92f4e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, vcs-type=git, io.k8s.display-name=CentOS Stream 8, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, release=754) 2024-02-07T12:53:54.991 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:54 smithi172 podman[86442]: 2024-02-07 12:53:54.864313395 +0000 UTC m=+1.009683072 container cleanup 7859b651e0d905ed705c599d03809e1f15c2621ab7d12be145131dff38a92f4e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6, vendor=Red Hat, Inc., version=8, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, release=754, com.redhat.component=centos-stream-container, RELEASE=HEAD, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux ) 2024-02-07T12:53:54.991 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:54 smithi172 bash[86442]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6 2024-02-07T12:53:55.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:54 smithi019 ceph-mon[82918]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:53:55.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:54 smithi019 ceph-mon[82918]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:53:55.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:54 smithi019 ceph-mon[82918]: failed to encode map e136 with expected crc 2024-02-07T12:53:55.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:54 smithi019 ceph-mon[82918]: osdmap e136: 8 total, 7 up, 8 in 2024-02-07T12:53:55.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:54 smithi019 ceph-mon[82918]: failed to encode map e136 with expected crc 2024-02-07T12:53:55.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:54 smithi019 ceph-mon[88137]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:53:55.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:54 smithi019 ceph-mon[88137]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:53:55.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:54 smithi019 ceph-mon[88137]: failed to encode map e136 with expected crc 2024-02-07T12:53:55.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:54 smithi019 ceph-mon[88137]: osdmap e136: 8 total, 7 up, 8 in 2024-02-07T12:53:55.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:54 smithi019 ceph-mon[88137]: failed to encode map e136 with expected crc 2024-02-07T12:53:55.369 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:55 smithi172 podman[86459]: 2024-02-07 12:53:55.11362492 +0000 UTC m=+0.751593636 container remove 7859b651e0d905ed705c599d03809e1f15c2621ab7d12be145131dff38a92f4e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, architecture=x86_64, maintainer=Guillaume Abrioux , ceph=True, RELEASE=HEAD, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, vcs-type=git, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-02-07T12:53:55.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:54 smithi172 ceph-mon[67636]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:53:55.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:54 smithi172 ceph-mon[67636]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:53:55.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:54 smithi172 ceph-mon[67636]: failed to encode map e136 with expected crc 2024-02-07T12:53:55.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:54 smithi172 ceph-mon[67636]: osdmap e136: 8 total, 7 up, 8 in 2024-02-07T12:53:55.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:54 smithi172 ceph-mon[67636]: failed to encode map e136 with expected crc 2024-02-07T12:53:55.738 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:55 smithi172 podman[86556]: 2024-02-07 12:53:55.344459257 +0000 UTC m=+0.018421252 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:53:55.738 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:55 smithi172 podman[86556]: 2024-02-07 12:53:55.483934634 +0000 UTC m=+0.157896616 container create da13a2af3dd62de0c01e24ae32f02b2402837d26d28029cb7a528495ee1bc00e (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.2) 2024-02-07T12:53:56.000 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:55 smithi172 podman[86556]: 2024-02-07 12:53:55.736909434 +0000 UTC m=+0.410871420 container init da13a2af3dd62de0c01e24ae32f02b2402837d26d28029cb7a528495ee1bc00e (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6-deactivate, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131) 2024-02-07T12:53:56.001 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:55 smithi172 podman[86556]: 2024-02-07 12:53:55.740452638 +0000 UTC m=+0.414414621 container start da13a2af3dd62de0c01e24ae32f02b2402837d26d28029cb7a528495ee1bc00e (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6-deactivate, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240131, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux ) 2024-02-07T12:53:56.001 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:55 smithi172 podman[86556]: 2024-02-07 12:53:55.82002711 +0000 UTC m=+0.493989118 container attach da13a2af3dd62de0c01e24ae32f02b2402837d26d28029cb7a528495ee1bc00e (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6-deactivate, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-02-07T12:53:56.276 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:55 smithi019 ceph-mon[88137]: failed to encode map e137 with expected crc 2024-02-07T12:53:56.276 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:55 smithi019 ceph-mon[88137]: osdmap e137: 8 total, 7 up, 8 in 2024-02-07T12:53:56.276 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:55 smithi019 ceph-mon[82918]: failed to encode map e137 with expected crc 2024-02-07T12:53:56.276 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:55 smithi019 ceph-mon[82918]: osdmap e137: 8 total, 7 up, 8 in 2024-02-07T12:53:56.325 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:55 smithi172 ceph-mon[67636]: failed to encode map e137 with expected crc 2024-02-07T12:53:56.325 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:55 smithi172 ceph-mon[67636]: osdmap e137: 8 total, 7 up, 8 in 2024-02-07T12:53:56.325 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:56 smithi172 podman[86556]: 2024-02-07 12:53:56.032063521 +0000 UTC m=+0.706025503 container died da13a2af3dd62de0c01e24ae32f02b2402837d26d28029cb7a528495ee1bc00e (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6-deactivate, org.label-schema.build-date=20240131, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-02-07T12:53:57.002 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:56 smithi172 podman[86582]: 2024-02-07 12:53:56.657626061 +0000 UTC m=+0.620513578 container remove da13a2af3dd62de0c01e24ae32f02b2402837d26d28029cb7a528495ee1bc00e (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6-deactivate, ceph=True, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-02-07T12:53:57.002 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:56 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.6.service: Deactivated successfully. 2024-02-07T12:53:57.002 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:56 smithi172 systemd[1]: Stopped Ceph osd.6 for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:53:57.003 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:56 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.6.service: Consumed 14.162s CPU time. 2024-02-07T12:53:57.003 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:56 smithi172 systemd[1]: Starting Ceph osd.6 for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:53:57.003 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:57 smithi019 ceph-mon[88137]: pgmap v236: 161 pgs: 4 active+undersized+degraded, 7 stale+active+clean, 11 active+undersized, 139 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 23/627 objects degraded (3.668%) 2024-02-07T12:53:57.004 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:57 smithi019 ceph-mon[82918]: pgmap v236: 161 pgs: 4 active+undersized+degraded, 7 stale+active+clean, 11 active+undersized, 139 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 23/627 objects degraded (3.668%) 2024-02-07T12:53:57.004 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:57 smithi019 ceph-mon[82918]: Health check failed: Degraded data redundancy: 23/627 objects degraded (3.668%), 4 pgs degraded (PG_DEGRADED) 2024-02-07T12:53:57.004 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:57 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:53:57.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:57 smithi019 ceph-mon[88137]: Health check failed: Degraded data redundancy: 23/627 objects degraded (3.668%), 4 pgs degraded (PG_DEGRADED) 2024-02-07T12:53:57.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:57 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:53:57.369 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:57 smithi172 podman[86687]: 2024-02-07 12:53:57.010815043 +0000 UTC m=+0.019805345 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:53:57.369 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:57 smithi172 podman[86687]: 2024-02-07 12:53:57.14196883 +0000 UTC m=+0.150959125 container create d1eb95867eee1e2c0992eca80aec823a0860c0943056aaabd3cb36bb13a17297 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6-activate, org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:53:57.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:57 smithi172 ceph-mon[67636]: pgmap v236: 161 pgs: 4 active+undersized+degraded, 7 stale+active+clean, 11 active+undersized, 139 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 23/627 objects degraded (3.668%) 2024-02-07T12:53:57.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:57 smithi172 ceph-mon[67636]: Health check failed: Degraded data redundancy: 23/627 objects degraded (3.668%), 4 pgs degraded (PG_DEGRADED) 2024-02-07T12:53:57.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:57 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:53:57.869 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:57 smithi172 podman[86687]: 2024-02-07 12:53:57.52099073 +0000 UTC m=+0.529981028 container init d1eb95867eee1e2c0992eca80aec823a0860c0943056aaabd3cb36bb13a17297 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6-activate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-02-07T12:53:57.869 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:57 smithi172 podman[86687]: 2024-02-07 12:53:57.524231879 +0000 UTC m=+0.533222178 container start d1eb95867eee1e2c0992eca80aec823a0860c0943056aaabd3cb36bb13a17297 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6-activate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240131, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.2) 2024-02-07T12:53:57.869 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:57 smithi172 podman[86687]: 2024-02-07 12:53:57.603215049 +0000 UTC m=+0.612205350 container attach d1eb95867eee1e2c0992eca80aec823a0860c0943056aaabd3cb36bb13a17297 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6-activate, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, io.buildah.version=1.33.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:53:58.291 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:53:58 smithi172 ceph-mon[67636]: pgmap v237: 161 pgs: 13 active+undersized+degraded, 2 stale+active+clean, 24 active+undersized, 122 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 53/627 objects degraded (8.453%) 2024-02-07T12:53:58.292 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6-activate[86701]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-02-07T12:53:58.292 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6-activate[86701]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-02-07T12:53:58.292 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:57 smithi172 bash[86687]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-02-07T12:53:58.292 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:57 smithi172 bash[86687]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-02-07T12:53:58.292 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:57 smithi172 bash[86687]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-02-07T12:53:58.292 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:57 smithi172 bash[86687]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-02-07T12:53:58.292 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:57 smithi172 bash[86687]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-02-07T12:53:58.292 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:57 smithi172 bash[86687]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-02-07T12:53:58.292 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:57 smithi172 bash[86687]: --> ceph-volume raw activate successful for osd ID: 6 2024-02-07T12:53:58.293 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6-activate[86701]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-02-07T12:53:58.293 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6-activate[86701]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-02-07T12:53:58.293 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6-activate[86701]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-02-07T12:53:58.293 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6-activate[86701]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-02-07T12:53:58.293 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6-activate[86701]: --> ceph-volume raw activate successful for osd ID: 6 2024-02-07T12:53:58.293 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:57 smithi172 podman[86687]: 2024-02-07 12:53:57.991953199 +0000 UTC m=+1.000943493 container died d1eb95867eee1e2c0992eca80aec823a0860c0943056aaabd3cb36bb13a17297 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6-activate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.2) 2024-02-07T12:53:58.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:53:58 smithi019 ceph-mon[88137]: pgmap v237: 161 pgs: 13 active+undersized+degraded, 2 stale+active+clean, 24 active+undersized, 122 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 53/627 objects degraded (8.453%) 2024-02-07T12:53:58.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:53:58 smithi019 ceph-mon[82918]: pgmap v237: 161 pgs: 13 active+undersized+degraded, 2 stale+active+clean, 24 active+undersized, 122 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 53/627 objects degraded (8.453%) 2024-02-07T12:53:58.965 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:58 smithi172 podman[86823]: 2024-02-07 12:53:58.640866238 +0000 UTC m=+0.643089480 container remove d1eb95867eee1e2c0992eca80aec823a0860c0943056aaabd3cb36bb13a17297 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6-activate, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-02-07T12:53:58.966 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:58 smithi172 podman[86889]: 2024-02-07 12:53:58.763205571 +0000 UTC m=+0.018087481 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:53:58.966 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:58 smithi172 podman[86889]: 2024-02-07 12:53:58.866776381 +0000 UTC m=+0.121658290 container create 219cf25240d51c78f8d79bc29de4964539d2e7f49d69d873b3bc7df0aedd8438 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True) 2024-02-07T12:53:59.369 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:59 smithi172 podman[86889]: 2024-02-07 12:53:59.069396957 +0000 UTC m=+0.324278864 container init 219cf25240d51c78f8d79bc29de4964539d2e7f49d69d873b3bc7df0aedd8438 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6, GIT_BRANCH=HEAD, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240131) 2024-02-07T12:53:59.369 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:59 smithi172 podman[86889]: 2024-02-07 12:53:59.072417475 +0000 UTC m=+0.327299382 container start 219cf25240d51c78f8d79bc29de4964539d2e7f49d69d873b3bc7df0aedd8438 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6, ceph=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-02-07T12:53:59.369 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:59 smithi172 ceph-osd[86907]: -- 172.21.15.172:0/2552201262 <== mon.2 v2:172.21.15.172:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56268a5996c0 con 0x56268a56b000 2024-02-07T12:53:59.369 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:59 smithi172 bash[86889]: 219cf25240d51c78f8d79bc29de4964539d2e7f49d69d873b3bc7df0aedd8438 2024-02-07T12:53:59.369 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:59 smithi172 systemd[1]: Started Ceph osd.6 for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:54:00.026 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:53:59 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:53:59.623+0000 7f87e7a167c0 -1 Falling back to public interface 2024-02-07T12:54:00.299 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:00 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:00.300 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:00 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:00.300 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:00 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:54:00.300 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:00 smithi172 ceph-mon[67636]: pgmap v238: 161 pgs: 18 active+undersized+degraded, 35 active+undersized, 108 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s; 74/627 objects degraded (11.802%) 2024-02-07T12:54:00.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:00 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:00.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:00 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:00.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:00 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:54:00.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:00 smithi019 ceph-mon[88137]: pgmap v238: 161 pgs: 18 active+undersized+degraded, 35 active+undersized, 108 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s; 74/627 objects degraded (11.802%) 2024-02-07T12:54:00.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:00 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:00.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:00 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:00.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:00 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:54:00.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:00 smithi019 ceph-mon[82918]: pgmap v238: 161 pgs: 18 active+undersized+degraded, 35 active+undersized, 108 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s; 74/627 objects degraded (11.802%) 2024-02-07T12:54:02.536 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:54:02 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:54:02.401+0000 7f87e7a167c0 -1 osd.6 0 read_superblock omap replica is missing. 2024-02-07T12:54:02.536 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:54:02 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:54:02.429+0000 7f87e7a167c0 -1 osd.6 135 log_to_monitors true 2024-02-07T12:54:03.227 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:02 smithi172 ceph-mon[67636]: pgmap v239: 161 pgs: 18 active+undersized+degraded, 35 active+undersized, 108 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 74/627 objects degraded (11.802%) 2024-02-07T12:54:03.227 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:02 smithi172 ceph-mon[67636]: from='osd.6 [v2:172.21.15.172:6816/3898009861,v1:172.21.15.172:6817/3898009861]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-02-07T12:54:03.227 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:02 smithi172 ceph-mon[67636]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-02-07T12:54:03.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:02 smithi019 ceph-mon[88137]: pgmap v239: 161 pgs: 18 active+undersized+degraded, 35 active+undersized, 108 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 74/627 objects degraded (11.802%) 2024-02-07T12:54:03.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:02 smithi019 ceph-mon[88137]: from='osd.6 [v2:172.21.15.172:6816/3898009861,v1:172.21.15.172:6817/3898009861]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-02-07T12:54:03.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:02 smithi019 ceph-mon[88137]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-02-07T12:54:03.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:02 smithi019 ceph-mon[82918]: pgmap v239: 161 pgs: 18 active+undersized+degraded, 35 active+undersized, 108 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 74/627 objects degraded (11.802%) 2024-02-07T12:54:03.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:02 smithi019 ceph-mon[82918]: from='osd.6 [v2:172.21.15.172:6816/3898009861,v1:172.21.15.172:6817/3898009861]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-02-07T12:54:03.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:02 smithi019 ceph-mon[82918]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-02-07T12:54:03.619 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:54:03 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:54:03.435+0000 7f87d4e6d700 -1 osd.6 135 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:54:04.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:03 smithi172 ceph-mon[67636]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-02-07T12:54:04.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:03 smithi172 ceph-mon[67636]: from='osd.6 [v2:172.21.15.172:6816/3898009861,v1:172.21.15.172:6817/3898009861]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:54:04.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:03 smithi172 ceph-mon[67636]: osdmap e138: 8 total, 7 up, 8 in 2024-02-07T12:54:04.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:03 smithi172 ceph-mon[67636]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:54:04.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:03 smithi172 ceph-mon[67636]: failed to encode map e138 with expected crc 2024-02-07T12:54:04.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:03 smithi172 ceph-mon[67636]: from='osd.6 ' entity='osd.6' 2024-02-07T12:54:04.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:03 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:04.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:03 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:04.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:54:03] "GET /metrics HTTP/1.1" 200 38275 "" "Prometheus/2.43.0" 2024-02-07T12:54:04.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:03 smithi019 ceph-mon[82918]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-02-07T12:54:04.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:03 smithi019 ceph-mon[82918]: from='osd.6 [v2:172.21.15.172:6816/3898009861,v1:172.21.15.172:6817/3898009861]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:54:04.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:03 smithi019 ceph-mon[82918]: osdmap e138: 8 total, 7 up, 8 in 2024-02-07T12:54:04.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:03 smithi019 ceph-mon[82918]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:54:04.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:03 smithi019 ceph-mon[82918]: failed to encode map e138 with expected crc 2024-02-07T12:54:04.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:03 smithi019 ceph-mon[82918]: from='osd.6 ' entity='osd.6' 2024-02-07T12:54:04.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:03 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:04.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:03 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:04.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:03 smithi019 ceph-mon[88137]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-02-07T12:54:04.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:03 smithi019 ceph-mon[88137]: from='osd.6 [v2:172.21.15.172:6816/3898009861,v1:172.21.15.172:6817/3898009861]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:54:04.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:03 smithi019 ceph-mon[88137]: osdmap e138: 8 total, 7 up, 8 in 2024-02-07T12:54:04.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:03 smithi019 ceph-mon[88137]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:54:04.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:03 smithi019 ceph-mon[88137]: failed to encode map e138 with expected crc 2024-02-07T12:54:04.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:03 smithi019 ceph-mon[88137]: from='osd.6 ' entity='osd.6' 2024-02-07T12:54:04.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:03 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:04.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:03 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:05.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:04 smithi019 ceph-mon[88137]: pgmap v241: 161 pgs: 18 active+undersized+degraded, 35 active+undersized, 108 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 114 B/s rd, 0 op/s; 74/627 objects degraded (11.802%) 2024-02-07T12:54:05.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:04 smithi019 ceph-mon[88137]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:54:05.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:04 smithi019 ceph-mon[88137]: failed to encode map e139 with expected crc 2024-02-07T12:54:05.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:04 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:54:05.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:04 smithi019 ceph-mon[88137]: osd.6 [v2:172.21.15.172:6816/3898009861,v1:172.21.15.172:6817/3898009861] boot 2024-02-07T12:54:05.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:04 smithi019 ceph-mon[88137]: osdmap e139: 8 total, 8 up, 8 in 2024-02-07T12:54:05.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:04 smithi019 ceph-mon[88137]: failed to encode map e139 with expected crc 2024-02-07T12:54:05.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:04 smithi019 ceph-mon[88137]: failed to encode map e139 with expected crc 2024-02-07T12:54:05.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:04 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:05.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:04 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:05.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:04 smithi019 ceph-mon[88137]: Health check update: Degraded data redundancy: 74/627 objects degraded (11.802%), 18 pgs degraded (PG_DEGRADED) 2024-02-07T12:54:05.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:04 smithi019 ceph-mon[82918]: pgmap v241: 161 pgs: 18 active+undersized+degraded, 35 active+undersized, 108 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 114 B/s rd, 0 op/s; 74/627 objects degraded (11.802%) 2024-02-07T12:54:05.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:04 smithi019 ceph-mon[82918]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:54:05.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:04 smithi019 ceph-mon[82918]: failed to encode map e139 with expected crc 2024-02-07T12:54:05.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:04 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:54:05.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:04 smithi019 ceph-mon[82918]: osd.6 [v2:172.21.15.172:6816/3898009861,v1:172.21.15.172:6817/3898009861] boot 2024-02-07T12:54:05.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:04 smithi019 ceph-mon[82918]: osdmap e139: 8 total, 8 up, 8 in 2024-02-07T12:54:05.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:04 smithi019 ceph-mon[82918]: failed to encode map e139 with expected crc 2024-02-07T12:54:05.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:04 smithi019 ceph-mon[82918]: failed to encode map e139 with expected crc 2024-02-07T12:54:05.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:04 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:05.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:04 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:05.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:04 smithi019 ceph-mon[82918]: Health check update: Degraded data redundancy: 74/627 objects degraded (11.802%), 18 pgs degraded (PG_DEGRADED) 2024-02-07T12:54:05.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:04 smithi172 ceph-mon[67636]: pgmap v241: 161 pgs: 18 active+undersized+degraded, 35 active+undersized, 108 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 114 B/s rd, 0 op/s; 74/627 objects degraded (11.802%) 2024-02-07T12:54:05.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:04 smithi172 ceph-mon[67636]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:54:05.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:04 smithi172 ceph-mon[67636]: failed to encode map e139 with expected crc 2024-02-07T12:54:05.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:04 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:54:05.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:04 smithi172 ceph-mon[67636]: osd.6 [v2:172.21.15.172:6816/3898009861,v1:172.21.15.172:6817/3898009861] boot 2024-02-07T12:54:05.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:04 smithi172 ceph-mon[67636]: osdmap e139: 8 total, 8 up, 8 in 2024-02-07T12:54:05.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:04 smithi172 ceph-mon[67636]: failed to encode map e139 with expected crc 2024-02-07T12:54:05.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:04 smithi172 ceph-mon[67636]: failed to encode map e139 with expected crc 2024-02-07T12:54:05.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:04 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:05.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:04 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:05.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:04 smithi172 ceph-mon[67636]: Health check update: Degraded data redundancy: 74/627 objects degraded (11.802%), 18 pgs degraded (PG_DEGRADED) 2024-02-07T12:54:06.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:06 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:06.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:06 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:54:06.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:06 smithi019 ceph-mon[88137]: failed to encode map e140 with expected crc 2024-02-07T12:54:06.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:06 smithi019 ceph-mon[88137]: osdmap e140: 8 total, 8 up, 8 in 2024-02-07T12:54:06.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:06 smithi019 ceph-mon[88137]: pgmap v244: 161 pgs: 9 peering, 15 active+undersized+degraded, 29 active+undersized, 108 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 60/627 objects degraded (9.569%) 2024-02-07T12:54:06.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:06 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:06.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:06 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:54:06.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:06 smithi019 ceph-mon[82918]: failed to encode map e140 with expected crc 2024-02-07T12:54:06.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:06 smithi019 ceph-mon[82918]: osdmap e140: 8 total, 8 up, 8 in 2024-02-07T12:54:06.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:06 smithi019 ceph-mon[82918]: pgmap v244: 161 pgs: 9 peering, 15 active+undersized+degraded, 29 active+undersized, 108 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 60/627 objects degraded (9.569%) 2024-02-07T12:54:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:06 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:06 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:54:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:06 smithi172 ceph-mon[67636]: failed to encode map e140 with expected crc 2024-02-07T12:54:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:06 smithi172 ceph-mon[67636]: osdmap e140: 8 total, 8 up, 8 in 2024-02-07T12:54:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:06 smithi172 ceph-mon[67636]: pgmap v244: 161 pgs: 9 peering, 15 active+undersized+degraded, 29 active+undersized, 108 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 60/627 objects degraded (9.569%) 2024-02-07T12:54:07.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:07 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:54:07.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:07 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:54:07.868 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:07 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:54:08.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:08 smithi019 ceph-mon[88137]: pgmap v245: 161 pgs: 9 peering, 9 active+undersized+degraded, 18 active+undersized, 125 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 34/627 objects degraded (5.423%) 2024-02-07T12:54:08.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:08 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:08.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:08 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:08.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:08 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:54:08.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:08 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:54:08.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:08 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:08.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:08 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:54:08.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:08 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:54:08.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:08 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:54:08.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:08 smithi019 ceph-mon[82918]: pgmap v245: 161 pgs: 9 peering, 9 active+undersized+degraded, 18 active+undersized, 125 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 34/627 objects degraded (5.423%) 2024-02-07T12:54:08.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:08 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:08.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:08 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:08.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:08 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:54:08.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:08 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:54:08.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:08 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:08.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:08 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:54:08.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:08 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:54:08.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:08 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:54:08.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:08 smithi172 ceph-mon[67636]: pgmap v245: 161 pgs: 9 peering, 9 active+undersized+degraded, 18 active+undersized, 125 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 34/627 objects degraded (5.423%) 2024-02-07T12:54:08.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:08 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:08.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:08 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:08.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:08 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:54:08.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:08 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:54:08.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:08 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:08.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:08 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:54:08.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:08 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:54:08.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:08 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:54:09.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:09 smithi019 ceph-mon[88137]: pgmap v246: 161 pgs: 9 peering, 152 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-02-07T12:54:09.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:09 smithi019 ceph-mon[88137]: pgmap v247: 161 pgs: 9 peering, 152 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 705 B/s rd, 0 op/s 2024-02-07T12:54:09.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:09 smithi019 ceph-mon[88137]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:54:09.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:09 smithi019 ceph-mon[88137]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:54:09.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:09 smithi019 ceph-mon[88137]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 60/627 objects degraded (9.569%), 15 pgs degraded) 2024-02-07T12:54:09.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:09 smithi019 ceph-mon[82918]: pgmap v246: 161 pgs: 9 peering, 152 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-02-07T12:54:09.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:09 smithi019 ceph-mon[82918]: pgmap v247: 161 pgs: 9 peering, 152 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 705 B/s rd, 0 op/s 2024-02-07T12:54:09.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:09 smithi019 ceph-mon[82918]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:54:09.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:09 smithi019 ceph-mon[82918]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:54:09.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:09 smithi019 ceph-mon[82918]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 60/627 objects degraded (9.569%), 15 pgs degraded) 2024-02-07T12:54:09.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:09 smithi172 ceph-mon[67636]: pgmap v246: 161 pgs: 9 peering, 152 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-02-07T12:54:09.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:09 smithi172 ceph-mon[67636]: pgmap v247: 161 pgs: 9 peering, 152 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 705 B/s rd, 0 op/s 2024-02-07T12:54:09.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:09 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:54:09.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:09 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:54:09.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:09 smithi172 ceph-mon[67636]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 60/627 objects degraded (9.569%), 15 pgs degraded) 2024-02-07T12:54:10.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:54:10.115+0000 7fb71d671700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:10.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:54:10.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:10.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:10.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:10.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:10.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:10.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:54:10.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:10.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:54:10.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:10.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:54:10.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:10.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:10.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:10.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:10.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:10.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:10.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:10.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:10.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:10.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:54:10.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:54:10.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:54:10.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:54:10.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:54:10.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:54:10.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:54:10.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:54:10.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:54:10.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:54:10.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:54:10.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:54:10.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:54:10.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:54:10.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:54:10.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:54:10.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:54:10.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:10.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:54:10.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:10.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:10.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:10.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:10.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:10.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:54:10.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:10.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:54:10.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:10.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:54:10.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:10.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:10.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:10.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:10.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:10.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:10.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:10.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:10.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:10.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:54:10.128+0000 7fb74ced9700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-02-07T12:54:11.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:54:11.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:11.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:11.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:11.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-07T12:54:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:54:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:10 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-07T12:54:11.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:10 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:54:11.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:10 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:11.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:10 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:11.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:10 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:11.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:10 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-07T12:54:12.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:11 smithi019 ceph-mon[88137]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:12.344 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:54:12.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:12.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:12.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:12.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:12.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:12.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:54:12.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:12.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:54:12.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:12.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:54:12.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:12.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:12.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:12.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:12.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:12.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:12.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:12.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:12.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:12.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:54:12.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:54:12.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:54:12.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:54:12.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:54:12.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:54:12.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:54:12.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:54:12.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:54:12.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:54:12.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:54:12.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:54:12.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:54:12.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:54:12.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:54:12.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:54:12.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:54:12.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:12.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:54:12.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:12.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:12.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:12.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:12.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:12.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:54:12.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:12.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:54:12.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:12.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:54:12.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:12.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:12.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:12.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:12.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:11 smithi019 ceph-mon[82918]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:12.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:54:12.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:12.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:12.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:12.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:12.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:12.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:54:12.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:12.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:54:12.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:12.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:54:12.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:12.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:12.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:12.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:12.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:12.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:12.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:12.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:12.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:12.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:54:12.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:54:12.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:54:12.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:54:12.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:54:12.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:54:12.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:54:12.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:54:12.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:54:12.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:54:12.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:54:12.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:54:12.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:54:12.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:54:12.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:54:12.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:54:12.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:54:12.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:12.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:54:12.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:12.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:12.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:12.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:12.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:12.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:54:12.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:12.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:54:12.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:12.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:54:12.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:12.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:12.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:12.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:12.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:12.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:12.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:12.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:12.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:12.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:11 smithi019 ceph-mon[82918]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-07T12:54:12.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:11 smithi019 ceph-mon[82918]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-02-07T12:54:12.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:11 smithi019 ceph-mon[82918]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 527 B/s rd, 0 op/s 2024-02-07T12:54:12.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:12.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:12.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:12.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:12.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:12.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:11 smithi019 ceph-mon[88137]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-07T12:54:12.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:11 smithi019 ceph-mon[88137]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-02-07T12:54:12.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:11 smithi019 ceph-mon[88137]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 527 B/s rd, 0 op/s 2024-02-07T12:54:12.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:11 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:12.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:54:12.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:12.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:12.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:12.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:12.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:12.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:54:12.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:12.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:54:12.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:12.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:54:12.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:12.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:12.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:12.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:12.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:12.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:12.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:12.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:12.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:12.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:54:12.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:54:12.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:54:12.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:54:12.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:54:12.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:54:12.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:54:12.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:54:12.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:54:12.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:54:12.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:54:12.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:54:12.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:54:12.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:54:12.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:54:12.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:54:12.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:54:12.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:12.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:54:12.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:12.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:12.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:12.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:12.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:12.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:54:12.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:12.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:54:12.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:12.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:54:12.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:12.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:12.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:12.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:12.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:12.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:12.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:12.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:12.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:12.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:11 smithi172 ceph-mon[67636]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-07T12:54:12.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:11 smithi172 ceph-mon[67636]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-02-07T12:54:12.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:11 smithi172 ceph-mon[67636]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 527 B/s rd, 0 op/s 2024-02-07T12:54:14.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:13 smithi019 ceph-mon[88137]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 450 B/s rd, 0 op/s 2024-02-07T12:54:14.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:13 smithi019 ceph-mon[82918]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 450 B/s rd, 0 op/s 2024-02-07T12:54:14.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:54:13] "GET /metrics HTTP/1.1" 200 38144 "" "Prometheus/2.43.0" 2024-02-07T12:54:14.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:13 smithi172 ceph-mon[67636]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 450 B/s rd, 0 op/s 2024-02-07T12:54:16.285 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:15 smithi019 ceph-mon[88137]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 980 B/s rd, 0 op/s 2024-02-07T12:54:16.285 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:15 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:16.285 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:15 smithi019 ceph-mon[82918]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 980 B/s rd, 0 op/s 2024-02-07T12:54:16.285 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:15 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:16.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:15 smithi172 ceph-mon[67636]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 980 B/s rd, 0 op/s 2024-02-07T12:54:16.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:15 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:18.186 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:17 smithi172 ceph-mon[67636]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 980 B/s rd, 0 op/s 2024-02-07T12:54:18.186 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:17 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:54:18.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:17 smithi019 ceph-mon[88137]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 980 B/s rd, 0 op/s 2024-02-07T12:54:18.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:17 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:54:18.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:17 smithi019 ceph-mon[82918]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 980 B/s rd, 0 op/s 2024-02-07T12:54:18.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:17 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:54:18.943 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:54:19.606 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:54:19.606 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (8m) 111s ago 30m 27.3M - 0.25.0 c8568f914cd2 fec9255f35e7 2024-02-07T12:54:19.607 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (8m) 16s ago 30m 84.8M - 9.4.12 a0ea5e3c46d9 aaffac63bc28 2024-02-07T12:54:19.607 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 unknown 111s ago 29m 63.5M - 3.5 e1d6a67b021e 5113aa77e3f2 2024-02-07T12:54:19.607 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283,8765 running (9m) 16s ago 34m 452M - 19.0.0-1252-g44458cce 79957b372307 5aa0c04fdfd7 2024-02-07T12:54:19.607 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (19m) 111s ago 36m 509M - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:54:19.607 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (6m) 111s ago 36m 53.6M 2048M 19.0.0-1252-g44458cce 79957b372307 8b1b0e21b1c5 2024-02-07T12:54:19.607 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (7m) 16s ago 35m 51.7M 2048M 19.0.0-1252-g44458cce 79957b372307 00f454bbac01 2024-02-07T12:54:19.607 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (5m) 111s ago 35m 43.3M 2048M 19.0.0-1252-g44458cce 79957b372307 6214003f6e08 2024-02-07T12:54:19.607 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (9m) 111s ago 31m 16.0M - 1.5.0 0da6a335fe13 c4bdcb1c4d49 2024-02-07T12:54:19.607 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (9m) 16s ago 31m 16.6M - 1.5.0 0da6a335fe13 04ea81f79d85 2024-02-07T12:54:19.607 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (4m) 111s ago 34m 56.0M 2993M 19.0.0-1252-g44458cce 79957b372307 78eddd34a903 2024-02-07T12:54:19.607 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (3m) 111s ago 34m 44.0M 2993M 19.0.0-1252-g44458cce 79957b372307 c37447e76c38 2024-02-07T12:54:19.607 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (2m) 111s ago 33m 50.2M 2993M 19.0.0-1252-g44458cce 79957b372307 90352897da85 2024-02-07T12:54:19.608 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (116s) 111s ago 33m 14.9M 2993M 19.0.0-1252-g44458cce 79957b372307 067cd6b3fd2f 2024-02-07T12:54:19.608 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (94s) 16s ago 33m 43.7M 3248M 19.0.0-1252-g44458cce 79957b372307 dbb47a8ad5ac 2024-02-07T12:54:19.608 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (57s) 16s ago 32m 43.1M 3248M 19.0.0-1252-g44458cce 79957b372307 d139d35fec57 2024-02-07T12:54:19.608 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (20s) 16s ago 32m 12.7M 3248M 19.0.0-1252-g44458cce 79957b372307 219cf25240d5 2024-02-07T12:54:19.608 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (31m) 16s ago 31m 64.7M 3248M 17.2.0 e1d6a67b021e b301e7c75d73 2024-02-07T12:54:19.608 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (8m) 16s ago 31m 71.5M - 2.43.0 a07b618ecd1d c9b607eea1be 2024-02-07T12:54:19.608 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (29m) 111s ago 29m 80.2M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:54:19.608 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (29m) 16s ago 29m 80.3M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:54:20.013 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:54:20.013 INFO:teuthology.orchestra.run.smithi019.stdout: "mon": { 2024-02-07T12:54:20.013 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 3 2024-02-07T12:54:20.013 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:54:20.013 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr": { 2024-02-07T12:54:20.013 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:54:20.013 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:54:20.013 INFO:teuthology.orchestra.run.smithi019.stdout: "osd": { 2024-02-07T12:54:20.013 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-02-07T12:54:20.014 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 7 2024-02-07T12:54:20.014 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:54:20.014 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw": { 2024-02-07T12:54:20.014 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:54:20.014 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:54:20.014 INFO:teuthology.orchestra.run.smithi019.stdout: "overall": { 2024-02-07T12:54:20.014 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-02-07T12:54:20.014 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 12 2024-02-07T12:54:20.014 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:54:20.014 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:54:20.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:19 smithi019 ceph-mon[88137]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:54:20.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:19 smithi019 ceph-mon[88137]: from='client.54514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:54:20.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:19 smithi019 ceph-mon[82918]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:54:20.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:19 smithi019 ceph-mon[82918]: from='client.54514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:54:20.359 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:54:20.359 INFO:teuthology.orchestra.run.smithi019.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:54:20.359 INFO:teuthology.orchestra.run.smithi019.stdout: "in_progress": true, 2024-02-07T12:54:20.359 INFO:teuthology.orchestra.run.smithi019.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-02-07T12:54:20.359 INFO:teuthology.orchestra.run.smithi019.stdout: "services_complete": [], 2024-02-07T12:54:20.359 INFO:teuthology.orchestra.run.smithi019.stdout: "progress": "7/8 daemons upgraded", 2024-02-07T12:54:20.359 INFO:teuthology.orchestra.run.smithi019.stdout: "message": "Currently upgrading osd daemons", 2024-02-07T12:54:20.359 INFO:teuthology.orchestra.run.smithi019.stdout: "is_paused": false 2024-02-07T12:54:20.359 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:54:20.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:19 smithi172 ceph-mon[67636]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:54:20.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:19 smithi172 ceph-mon[67636]: from='client.54514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:54:21.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:20 smithi019 ceph-mon[82918]: from='client.54520 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:54:21.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:20 smithi019 ceph-mon[82918]: from='client.44586 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:54:21.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:20 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/2656867636' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:21.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:20 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:21.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:20 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:54:21.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:20 smithi019 ceph-mon[88137]: from='client.54520 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:54:21.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:20 smithi019 ceph-mon[88137]: from='client.44586 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:54:21.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:20 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/2656867636' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:21.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:20 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:21.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:20 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:54:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:20 smithi172 ceph-mon[67636]: from='client.54520 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:54:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:20 smithi172 ceph-mon[67636]: from='client.44586 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:54:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:20 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/2656867636' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:20 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:20 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:54:22.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:21 smithi019 ceph-mon[82918]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:54:22.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:21 smithi019 ceph-mon[82918]: from='client.54538 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:54:22.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:21 smithi019 ceph-mon[88137]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:54:22.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:21 smithi019 ceph-mon[88137]: from='client.54538 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:54:22.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:21 smithi172 ceph-mon[67636]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:54:22.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:21 smithi172 ceph-mon[67636]: from='client.54538 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:54:24.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:23 smithi019 ceph-mon[88137]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:54:24.370 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:23 smithi019 ceph-mon[82918]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:54:24.370 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:54:23] "GET /metrics HTTP/1.1" 200 38285 "" "Prometheus/2.43.0" 2024-02-07T12:54:24.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:23 smithi172 ceph-mon[67636]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:54:26.038 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:25 smithi172 ceph-mon[67636]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:54:26.038 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:25 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-07T12:54:26.296 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:25 smithi019 ceph-mon[88137]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:54:26.296 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:25 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-07T12:54:26.296 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:25 smithi019 ceph-mon[82918]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:54:26.296 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:25 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-07T12:54:27.282 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:26 smithi172 ceph-mon[67636]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-07T12:54:27.282 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:26 smithi172 ceph-mon[67636]: Upgrade: osd.7 is safe to restart 2024-02-07T12:54:27.282 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:26 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:27.282 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:26 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-02-07T12:54:27.282 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:26 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:54:27.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:26 smithi019 ceph-mon[82918]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-07T12:54:27.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:26 smithi019 ceph-mon[82918]: Upgrade: osd.7 is safe to restart 2024-02-07T12:54:27.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:26 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:27.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:26 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-02-07T12:54:27.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:26 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:54:27.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:26 smithi019 ceph-mon[88137]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-07T12:54:27.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:26 smithi019 ceph-mon[88137]: Upgrade: osd.7 is safe to restart 2024-02-07T12:54:27.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:26 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:27.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:26 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-02-07T12:54:27.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:26 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:54:28.232 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:27 smithi172 ceph-mon[67636]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:54:28.232 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:27 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:54:28.232 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:27 smithi172 ceph-mon[67636]: Upgrade: Updating osd.7 2024-02-07T12:54:28.232 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:27 smithi172 ceph-mon[67636]: Deploying daemon osd.7 on smithi172 2024-02-07T12:54:28.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:27 smithi019 ceph-mon[82918]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:54:28.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:27 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:54:28.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:27 smithi019 ceph-mon[82918]: Upgrade: Updating osd.7 2024-02-07T12:54:28.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:27 smithi019 ceph-mon[82918]: Deploying daemon osd.7 on smithi172 2024-02-07T12:54:28.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:27 smithi019 ceph-mon[88137]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:54:28.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:27 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:54:28.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:27 smithi019 ceph-mon[88137]: Upgrade: Updating osd.7 2024-02-07T12:54:28.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:27 smithi019 ceph-mon[88137]: Deploying daemon osd.7 on smithi172 2024-02-07T12:54:30.321 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:29 smithi172 ceph-mon[67636]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:54:30.321 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:30 smithi172 systemd[1]: Stopping Ceph osd.7 for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:54:30.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:29 smithi019 ceph-mon[82918]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:54:30.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:29 smithi019 ceph-mon[88137]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:54:30.578 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:30 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[37568]: 2024-02-07T12:54:30.317+0000 7f11d34b9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:54:30.578 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:30 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[37568]: 2024-02-07T12:54:30.317+0000 7f11d34b9700 -1 osd.7 140 *** Got signal Terminated *** 2024-02-07T12:54:30.578 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:30 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[37568]: 2024-02-07T12:54:30.317+0000 7f11d34b9700 -1 osd.7 140 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:54:31.320 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:30 smithi172 ceph-mon[67636]: osd.7 marked itself down and dead 2024-02-07T12:54:31.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:30 smithi019 ceph-mon[88137]: osd.7 marked itself down and dead 2024-02-07T12:54:31.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:30 smithi019 ceph-mon[82918]: osd.7 marked itself down and dead 2024-02-07T12:54:31.596 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:31 smithi172 podman[91440]: 2024-02-07 12:54:31.318049914 +0000 UTC m=+1.092010825 container died b301e7c75d73ce949a8b332fd13af9268e0015673e6db403466298f2d023e76f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, io.buildah.version=1.19.8, io.openshift.expose-services=, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, name=centos-stream, vendor=Red Hat, Inc., ceph=True, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, release=754, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container) 2024-02-07T12:54:31.869 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:31 smithi172 podman[91440]: 2024-02-07 12:54:31.684813186 +0000 UTC m=+1.458774095 container cleanup b301e7c75d73ce949a8b332fd13af9268e0015673e6db403466298f2d023e76f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, vcs-type=git, io.k8s.display-name=CentOS Stream 8, ceph=True, GIT_CLEAN=True, vendor=Red Hat, Inc., release=754, version=8, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.buildah.version=1.19.8) 2024-02-07T12:54:31.869 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:31 smithi172 bash[91440]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7 2024-02-07T12:54:32.313 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:31 smithi172 podman[91458]: 2024-02-07 12:54:31.978388463 +0000 UTC m=+0.663313092 container remove b301e7c75d73ce949a8b332fd13af9268e0015673e6db403466298f2d023e76f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7, architecture=x86_64, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., vcs-type=git, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-02-07T12:54:32.313 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:31 smithi172 ceph-mon[67636]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:54:32.314 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:31 smithi172 ceph-mon[67636]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:54:32.314 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:31 smithi172 ceph-mon[67636]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-02-07T12:54:32.314 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:31 smithi172 ceph-mon[67636]: osdmap e141: 8 total, 7 up, 8 in 2024-02-07T12:54:32.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:31 smithi019 ceph-mon[88137]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:54:32.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:31 smithi019 ceph-mon[88137]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:54:32.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:31 smithi019 ceph-mon[88137]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-02-07T12:54:32.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:31 smithi019 ceph-mon[88137]: osdmap e141: 8 total, 7 up, 8 in 2024-02-07T12:54:32.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:31 smithi019 ceph-mon[82918]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:54:32.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:31 smithi019 ceph-mon[82918]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:54:32.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:31 smithi019 ceph-mon[82918]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-02-07T12:54:32.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:31 smithi019 ceph-mon[82918]: osdmap e141: 8 total, 7 up, 8 in 2024-02-07T12:54:32.599 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:32 smithi172 podman[91551]: 2024-02-07 12:54:32.216353595 +0000 UTC m=+0.019363472 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:54:32.600 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:32 smithi172 podman[91551]: 2024-02-07 12:54:32.362272586 +0000 UTC m=+0.165282459 container create 299d36b011a68511728b662a19b85ed852979a72721c5e6db9e699a6f6f22bfe (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-02-07T12:54:32.869 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:32 smithi172 podman[91551]: 2024-02-07 12:54:32.598365607 +0000 UTC m=+0.401375477 container init 299d36b011a68511728b662a19b85ed852979a72721c5e6db9e699a6f6f22bfe (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7-deactivate, io.buildah.version=1.33.2, GIT_BRANCH=HEAD, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-02-07T12:54:32.869 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:32 smithi172 podman[91551]: 2024-02-07 12:54:32.601567485 +0000 UTC m=+0.404577357 container start 299d36b011a68511728b662a19b85ed852979a72721c5e6db9e699a6f6f22bfe (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.2) 2024-02-07T12:54:32.869 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:32 smithi172 podman[91551]: 2024-02-07 12:54:32.71517113 +0000 UTC m=+0.518181028 container attach 299d36b011a68511728b662a19b85ed852979a72721c5e6db9e699a6f6f22bfe (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.2) 2024-02-07T12:54:33.195 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:32 smithi172 podman[91551]: 2024-02-07 12:54:32.891240685 +0000 UTC m=+0.694250550 container died 299d36b011a68511728b662a19b85ed852979a72721c5e6db9e699a6f6f22bfe (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7-deactivate, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240131, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:54:33.195 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:32 smithi172 ceph-mon[67636]: osdmap e142: 8 total, 7 up, 8 in 2024-02-07T12:54:33.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:32 smithi019 ceph-mon[88137]: osdmap e142: 8 total, 7 up, 8 in 2024-02-07T12:54:33.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:32 smithi019 ceph-mon[82918]: osdmap e142: 8 total, 7 up, 8 in 2024-02-07T12:54:33.455 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:33 smithi172 podman[91577]: 2024-02-07 12:54:33.435629378 +0000 UTC m=+0.538904830 container remove 299d36b011a68511728b662a19b85ed852979a72721c5e6db9e699a6f6f22bfe (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-02-07T12:54:33.869 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:33 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.7.service: Deactivated successfully. 2024-02-07T12:54:33.869 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:33 smithi172 systemd[1]: Stopped Ceph osd.7 for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:54:33.869 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:33 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.7.service: Consumed 16.122s CPU time. 2024-02-07T12:54:33.869 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:33 smithi172 systemd[1]: Starting Ceph osd.7 for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:54:34.199 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:33 smithi172 podman[91684]: 2024-02-07 12:54:33.773151668 +0000 UTC m=+0.018074517 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:54:34.199 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:33 smithi172 podman[91684]: 2024-02-07 12:54:33.893896954 +0000 UTC m=+0.138819834 container create 31658c5938a31b397a1f79456825b0ba2a01d77df4e08c4f3c548ae8629be76d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7-activate, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:54:34.200 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:33 smithi172 ceph-mon[67636]: pgmap v261: 161 pgs: 1 active+undersized+degraded, 23 peering, 6 active+undersized, 131 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 1/627 objects degraded (0.159%) 2024-02-07T12:54:34.200 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:33 smithi172 ceph-mon[67636]: Health check failed: Reduced data availability: 3 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-02-07T12:54:34.200 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:33 smithi172 ceph-mon[67636]: Health check failed: Degraded data redundancy: 1/627 objects degraded (0.159%), 1 pg degraded (PG_DEGRADED) 2024-02-07T12:54:34.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:54:33] "GET /metrics HTTP/1.1" 200 38285 "" "Prometheus/2.43.0" 2024-02-07T12:54:34.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:33 smithi019 ceph-mon[82918]: pgmap v261: 161 pgs: 1 active+undersized+degraded, 23 peering, 6 active+undersized, 131 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 1/627 objects degraded (0.159%) 2024-02-07T12:54:34.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:33 smithi019 ceph-mon[82918]: Health check failed: Reduced data availability: 3 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-02-07T12:54:34.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:33 smithi019 ceph-mon[82918]: Health check failed: Degraded data redundancy: 1/627 objects degraded (0.159%), 1 pg degraded (PG_DEGRADED) 2024-02-07T12:54:34.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:33 smithi019 ceph-mon[88137]: pgmap v261: 161 pgs: 1 active+undersized+degraded, 23 peering, 6 active+undersized, 131 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 1/627 objects degraded (0.159%) 2024-02-07T12:54:34.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:33 smithi019 ceph-mon[88137]: Health check failed: Reduced data availability: 3 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-02-07T12:54:34.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:33 smithi019 ceph-mon[88137]: Health check failed: Degraded data redundancy: 1/627 objects degraded (0.159%), 1 pg degraded (PG_DEGRADED) 2024-02-07T12:54:34.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:34 smithi172 podman[91684]: 2024-02-07 12:54:34.197816125 +0000 UTC m=+0.442738980 container init 31658c5938a31b397a1f79456825b0ba2a01d77df4e08c4f3c548ae8629be76d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7-activate, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS) 2024-02-07T12:54:34.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:34 smithi172 podman[91684]: 2024-02-07 12:54:34.201636547 +0000 UTC m=+0.446559385 container start 31658c5938a31b397a1f79456825b0ba2a01d77df4e08c4f3c548ae8629be76d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7-activate, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:54:34.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:34 smithi172 podman[91684]: 2024-02-07 12:54:34.297733233 +0000 UTC m=+0.542656087 container attach 31658c5938a31b397a1f79456825b0ba2a01d77df4e08c4f3c548ae8629be76d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7-activate, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240131, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux ) 2024-02-07T12:54:34.961 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:34 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7-activate[91699]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-02-07T12:54:34.961 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:34 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7-activate[91699]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-02-07T12:54:34.961 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:34 smithi172 bash[91684]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-02-07T12:54:34.961 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:34 smithi172 bash[91684]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-02-07T12:54:34.961 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:34 smithi172 bash[91684]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-02-07T12:54:34.962 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:34 smithi172 bash[91684]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-02-07T12:54:34.962 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:34 smithi172 bash[91684]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-02-07T12:54:34.962 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:34 smithi172 bash[91684]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-02-07T12:54:34.962 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:34 smithi172 bash[91684]: --> ceph-volume raw activate successful for osd ID: 7 2024-02-07T12:54:34.962 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:34 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7-activate[91699]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-02-07T12:54:34.962 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:34 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7-activate[91699]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-02-07T12:54:34.962 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:34 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7-activate[91699]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-02-07T12:54:34.962 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:34 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7-activate[91699]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-02-07T12:54:34.962 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:34 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7-activate[91699]: --> ceph-volume raw activate successful for osd ID: 7 2024-02-07T12:54:34.962 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:34 smithi172 podman[91684]: 2024-02-07 12:54:34.671814628 +0000 UTC m=+0.916737467 container died 31658c5938a31b397a1f79456825b0ba2a01d77df4e08c4f3c548ae8629be76d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7-activate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.2, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:54:35.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:35 smithi172 podman[91825]: 2024-02-07 12:54:35.327561969 +0000 UTC m=+0.650310400 container remove 31658c5938a31b397a1f79456825b0ba2a01d77df4e08c4f3c548ae8629be76d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7-activate, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-02-07T12:54:35.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:35 smithi172 podman[91891]: 2024-02-07 12:54:35.448251655 +0000 UTC m=+0.019305264 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:54:35.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:35 smithi172 podman[91891]: 2024-02-07 12:54:35.553670865 +0000 UTC m=+0.124724470 container create 75dab2b6606bcc90dbf7e7367bc75136f1dd8d09860d216679761d706aa50a9d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:54:35.983 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:35 smithi172 podman[91891]: 2024-02-07 12:54:35.789516947 +0000 UTC m=+0.360570567 container init 75dab2b6606bcc90dbf7e7367bc75136f1dd8d09860d216679761d706aa50a9d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131, io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2) 2024-02-07T12:54:35.983 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:35 smithi172 podman[91891]: 2024-02-07 12:54:35.793980536 +0000 UTC m=+0.365034141 container start 75dab2b6606bcc90dbf7e7367bc75136f1dd8d09860d216679761d706aa50a9d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, org.label-schema.build-date=20240131, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-02-07T12:54:35.983 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:35 smithi172 ceph-osd[91910]: -- 172.21.15.172:0/1821224100 <== mon.2 v2:172.21.15.172:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e4ba7f56c0 con 0x55e4ba7c6c00 2024-02-07T12:54:35.983 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:35 smithi172 bash[91891]: 75dab2b6606bcc90dbf7e7367bc75136f1dd8d09860d216679761d706aa50a9d 2024-02-07T12:54:35.983 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:35 smithi172 systemd[1]: Started Ceph osd.7 for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:54:36.305 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:35 smithi019 ceph-mon[88137]: pgmap v262: 161 pgs: 10 active+undersized+degraded, 23 peering, 26 active+undersized, 102 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 24/627 objects degraded (3.828%) 2024-02-07T12:54:36.305 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:35 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:36.305 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:35 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:54:36.305 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:35 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:36.305 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:35 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:36.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:35 smithi019 ceph-mon[82918]: pgmap v262: 161 pgs: 10 active+undersized+degraded, 23 peering, 26 active+undersized, 102 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 24/627 objects degraded (3.828%) 2024-02-07T12:54:36.306 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:35 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:36.306 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:35 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:54:36.306 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:35 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:36.306 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:35 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:36.340 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:35 smithi172 ceph-mon[67636]: pgmap v262: 161 pgs: 10 active+undersized+degraded, 23 peering, 26 active+undersized, 102 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 24/627 objects degraded (3.828%) 2024-02-07T12:54:36.340 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:35 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:36.340 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:35 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:54:36.341 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:35 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:36.341 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:35 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:36.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:36 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:54:36.337+0000 7f1bba8827c0 -1 Falling back to public interface 2024-02-07T12:54:37.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:36 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:54:37.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:36 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:54:37.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:36 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:54:38.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:37 smithi019 ceph-mon[88137]: pgmap v263: 161 pgs: 22 active+undersized+degraded, 10 peering, 37 active+undersized, 92 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 9.1 KiB/s rd, 0 B/s wr, 16 op/s; 67/627 objects degraded (10.686%) 2024-02-07T12:54:38.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:37 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:54:38.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:37 smithi019 ceph-mon[82918]: pgmap v263: 161 pgs: 22 active+undersized+degraded, 10 peering, 37 active+undersized, 92 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 9.1 KiB/s rd, 0 B/s wr, 16 op/s; 67/627 objects degraded (10.686%) 2024-02-07T12:54:38.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:37 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:54:38.368 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:37 smithi172 ceph-mon[67636]: pgmap v263: 161 pgs: 22 active+undersized+degraded, 10 peering, 37 active+undersized, 92 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 9.1 KiB/s rd, 0 B/s wr, 16 op/s; 67/627 objects degraded (10.686%) 2024-02-07T12:54:38.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:37 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:54:39.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:39 smithi019 ceph-mon[88137]: pgmap v264: 161 pgs: 27 active+undersized+degraded, 42 active+undersized, 92 active+clean; 585 KiB data, 233 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s rd, 0 B/s wr, 103 op/s; 83/627 objects degraded (13.238%) 2024-02-07T12:54:39.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:39 smithi019 ceph-mon[88137]: Health check update: Degraded data redundancy: 83/627 objects degraded (13.238%), 27 pgs degraded (PG_DEGRADED) 2024-02-07T12:54:39.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:39 smithi019 ceph-mon[88137]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2024-02-07T12:54:39.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:39 smithi019 ceph-mon[82918]: pgmap v264: 161 pgs: 27 active+undersized+degraded, 42 active+undersized, 92 active+clean; 585 KiB data, 233 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s rd, 0 B/s wr, 103 op/s; 83/627 objects degraded (13.238%) 2024-02-07T12:54:39.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:39 smithi019 ceph-mon[82918]: Health check update: Degraded data redundancy: 83/627 objects degraded (13.238%), 27 pgs degraded (PG_DEGRADED) 2024-02-07T12:54:39.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:39 smithi019 ceph-mon[82918]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2024-02-07T12:54:39.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:39 smithi172 ceph-mon[67636]: pgmap v264: 161 pgs: 27 active+undersized+degraded, 42 active+undersized, 92 active+clean; 585 KiB data, 233 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s rd, 0 B/s wr, 103 op/s; 83/627 objects degraded (13.238%) 2024-02-07T12:54:39.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:39 smithi172 ceph-mon[67636]: Health check update: Degraded data redundancy: 83/627 objects degraded (13.238%), 27 pgs degraded (PG_DEGRADED) 2024-02-07T12:54:39.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:39 smithi172 ceph-mon[67636]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2024-02-07T12:54:39.369 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:39 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:54:39.043+0000 7f1bba8827c0 -1 osd.7 0 read_superblock omap replica is missing. 2024-02-07T12:54:39.369 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:39 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:54:39.078+0000 7f1bba8827c0 -1 osd.7 140 log_to_monitors true 2024-02-07T12:54:40.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:40 smithi019 ceph-mon[88137]: from='osd.7 [v2:172.21.15.172:6824/2869526670,v1:172.21.15.172:6825/2869526670]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-07T12:54:40.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:40 smithi019 ceph-mon[88137]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-07T12:54:40.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:40 smithi019 ceph-mon[82918]: from='osd.7 [v2:172.21.15.172:6824/2869526670,v1:172.21.15.172:6825/2869526670]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-07T12:54:40.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:40 smithi019 ceph-mon[82918]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-07T12:54:40.369 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:54:40 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:54:40.077+0000 7f1babce1700 -1 osd.7 140 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:54:40.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:40 smithi172 ceph-mon[67636]: from='osd.7 [v2:172.21.15.172:6824/2869526670,v1:172.21.15.172:6825/2869526670]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-07T12:54:40.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:40 smithi172 ceph-mon[67636]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-07T12:54:41.061 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:41 smithi172 ceph-mon[67636]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-02-07T12:54:41.061 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:41 smithi172 ceph-mon[67636]: from='osd.7 [v2:172.21.15.172:6824/2869526670,v1:172.21.15.172:6825/2869526670]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:54:41.061 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:41 smithi172 ceph-mon[67636]: osdmap e143: 8 total, 7 up, 8 in 2024-02-07T12:54:41.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:41 smithi019 ceph-mon[82918]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-02-07T12:54:41.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:41 smithi019 ceph-mon[82918]: from='osd.7 [v2:172.21.15.172:6824/2869526670,v1:172.21.15.172:6825/2869526670]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:54:41.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:41 smithi019 ceph-mon[82918]: osdmap e143: 8 total, 7 up, 8 in 2024-02-07T12:54:41.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:41 smithi019 ceph-mon[82918]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:54:41.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:41 smithi019 ceph-mon[82918]: pgmap v266: 161 pgs: 27 active+undersized+degraded, 42 active+undersized, 92 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 0 B/s wr, 139 op/s; 83/627 objects degraded (13.238%) 2024-02-07T12:54:41.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:41 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:41.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:41 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:41.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:41 smithi019 ceph-mon[88137]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-02-07T12:54:41.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:41 smithi019 ceph-mon[88137]: from='osd.7 [v2:172.21.15.172:6824/2869526670,v1:172.21.15.172:6825/2869526670]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:54:41.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:41 smithi019 ceph-mon[88137]: osdmap e143: 8 total, 7 up, 8 in 2024-02-07T12:54:41.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:41 smithi019 ceph-mon[88137]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:54:41.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:41 smithi019 ceph-mon[88137]: pgmap v266: 161 pgs: 27 active+undersized+degraded, 42 active+undersized, 92 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 0 B/s wr, 139 op/s; 83/627 objects degraded (13.238%) 2024-02-07T12:54:41.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:41 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:41.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:41 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:41.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:41 smithi172 ceph-mon[67636]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-02-07T12:54:41.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:41 smithi172 ceph-mon[67636]: pgmap v266: 161 pgs: 27 active+undersized+degraded, 42 active+undersized, 92 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 0 B/s wr, 139 op/s; 83/627 objects degraded (13.238%) 2024-02-07T12:54:41.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:41 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:41.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:41 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:42.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:42 smithi019 ceph-mon[88137]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:54:42.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:42 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:54:42.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:42 smithi019 ceph-mon[88137]: osd.7 [v2:172.21.15.172:6824/2869526670,v1:172.21.15.172:6825/2869526670] boot 2024-02-07T12:54:42.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:42 smithi019 ceph-mon[88137]: osdmap e144: 8 total, 8 up, 8 in 2024-02-07T12:54:42.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:42 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:42.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:42 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:42 smithi019 ceph-mon[82918]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:54:42.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:42 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:54:42.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:42 smithi019 ceph-mon[82918]: osd.7 [v2:172.21.15.172:6824/2869526670,v1:172.21.15.172:6825/2869526670] boot 2024-02-07T12:54:42.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:42 smithi019 ceph-mon[82918]: osdmap e144: 8 total, 8 up, 8 in 2024-02-07T12:54:42.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:42 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:42.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:42 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:42.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:42 smithi172 ceph-mon[67636]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:54:42.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:42 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:54:42.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:42 smithi172 ceph-mon[67636]: osd.7 [v2:172.21.15.172:6824/2869526670,v1:172.21.15.172:6825/2869526670] boot 2024-02-07T12:54:42.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:42 smithi172 ceph-mon[67636]: osdmap e144: 8 total, 8 up, 8 in 2024-02-07T12:54:42.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:42 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:42.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:42 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:43.142 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:43 smithi172 ceph-mon[67636]: osdmap e145: 8 total, 8 up, 8 in 2024-02-07T12:54:43.142 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:43 smithi172 ceph-mon[67636]: pgmap v269: 161 pgs: 21 peering, 19 active+undersized+degraded, 29 active+undersized, 92 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 123 KiB/s rd, 0 B/s wr, 205 op/s; 67/627 objects degraded (10.686%) 2024-02-07T12:54:43.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:43 smithi019 ceph-mon[88137]: osdmap e145: 8 total, 8 up, 8 in 2024-02-07T12:54:43.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:43 smithi019 ceph-mon[88137]: pgmap v269: 161 pgs: 21 peering, 19 active+undersized+degraded, 29 active+undersized, 92 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 123 KiB/s rd, 0 B/s wr, 205 op/s; 67/627 objects degraded (10.686%) 2024-02-07T12:54:43.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:43 smithi019 ceph-mon[82918]: osdmap e145: 8 total, 8 up, 8 in 2024-02-07T12:54:43.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:43 smithi019 ceph-mon[82918]: pgmap v269: 161 pgs: 21 peering, 19 active+undersized+degraded, 29 active+undersized, 92 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 123 KiB/s rd, 0 B/s wr, 205 op/s; 67/627 objects degraded (10.686%) 2024-02-07T12:54:44.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:54:43] "GET /metrics HTTP/1.1" 200 38363 "" "Prometheus/2.43.0" 2024-02-07T12:54:44.615 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:44 smithi172 ceph-mon[67636]: Health check update: Degraded data redundancy: 48/627 objects degraded (7.656%), 11 pgs degraded (PG_DEGRADED) 2024-02-07T12:54:44.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:44 smithi019 ceph-mon[88137]: Health check update: Degraded data redundancy: 48/627 objects degraded (7.656%), 11 pgs degraded (PG_DEGRADED) 2024-02-07T12:54:44.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:44 smithi019 ceph-mon[82918]: Health check update: Degraded data redundancy: 48/627 objects degraded (7.656%), 11 pgs degraded (PG_DEGRADED) 2024-02-07T12:54:46.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:45 smithi019 ceph-mon[88137]: pgmap v270: 161 pgs: 21 peering, 11 active+undersized+degraded, 15 active+undersized, 114 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 0 B/s wr, 91 op/s; 48/627 objects degraded (7.656%); 0 B/s, 1 objects/s recovering 2024-02-07T12:54:46.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:45 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:46.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:45 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:46.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:45 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:54:46.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:45 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:54:46.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:45 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:46.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:45 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:54:46.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:45 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:54:46.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:45 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:54:46.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:45 smithi019 ceph-mon[82918]: pgmap v270: 161 pgs: 21 peering, 11 active+undersized+degraded, 15 active+undersized, 114 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 0 B/s wr, 91 op/s; 48/627 objects degraded (7.656%); 0 B/s, 1 objects/s recovering 2024-02-07T12:54:46.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:45 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:46.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:45 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:46.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:45 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:54:46.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:45 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:54:46.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:45 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:46.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:45 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:54:46.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:45 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:54:46.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:45 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:54:46.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:45 smithi172 ceph-mon[67636]: pgmap v270: 161 pgs: 21 peering, 11 active+undersized+degraded, 15 active+undersized, 114 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 0 B/s wr, 91 op/s; 48/627 objects degraded (7.656%); 0 B/s, 1 objects/s recovering 2024-02-07T12:54:46.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:45 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:46.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:45 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:46.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:45 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:54:46.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:45 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:54:46.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:45 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:46.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:45 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:54:46.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:45 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:54:46.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:45 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:54:47.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:46 smithi019 ceph-mon[88137]: pgmap v271: 161 pgs: 21 peering, 3 active+undersized+degraded, 1 active+undersized, 136 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s rd, 0 B/s wr, 115 op/s; 11/627 objects degraded (1.754%); 0 B/s, 7 objects/s recovering 2024-02-07T12:54:47.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:46 smithi019 ceph-mon[88137]: pgmap v272: 161 pgs: 21 peering, 3 active+undersized+degraded, 1 active+undersized, 136 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 57 op/s; 11/627 objects degraded (1.754%); 0 B/s, 7 objects/s recovering 2024-02-07T12:54:47.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:46 smithi019 ceph-mon[88137]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:54:47.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:46 smithi019 ceph-mon[88137]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:54:47.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:46 smithi019 ceph-mon[82918]: pgmap v271: 161 pgs: 21 peering, 3 active+undersized+degraded, 1 active+undersized, 136 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s rd, 0 B/s wr, 115 op/s; 11/627 objects degraded (1.754%); 0 B/s, 7 objects/s recovering 2024-02-07T12:54:47.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:46 smithi019 ceph-mon[82918]: pgmap v272: 161 pgs: 21 peering, 3 active+undersized+degraded, 1 active+undersized, 136 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 57 op/s; 11/627 objects degraded (1.754%); 0 B/s, 7 objects/s recovering 2024-02-07T12:54:47.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:46 smithi019 ceph-mon[82918]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:54:47.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:46 smithi019 ceph-mon[82918]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:54:47.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:54:47.051+0000 7fb71d671700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:47.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:54:47.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:47.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:47.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:47.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:47.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:47.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:54:47.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:47.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:54:47.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:47.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:54:47.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:47.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:47.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:47.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:47.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:47.346 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:47.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:47.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:47.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:47.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:54:47.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:54:47.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:54:47.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:54:47.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:54:47.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:54:47.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:54:47.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:54:47.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:54:47.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:54:47.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:54:47.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:54:47.347 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:54:47.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:54:47.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:54:47.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:54:47.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:54:47.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:47.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:54:47.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:47.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:47.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:47.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:47.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:47.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:54:47.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:47.348 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:54:47.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:47.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:54:47.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:47.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:47.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:47.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:47.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:47.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:47.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:47.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:47.349 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:47.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:46 smithi172 ceph-mon[67636]: pgmap v271: 161 pgs: 21 peering, 3 active+undersized+degraded, 1 active+undersized, 136 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s rd, 0 B/s wr, 115 op/s; 11/627 objects degraded (1.754%); 0 B/s, 7 objects/s recovering 2024-02-07T12:54:47.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:46 smithi172 ceph-mon[67636]: pgmap v272: 161 pgs: 21 peering, 3 active+undersized+degraded, 1 active+undersized, 136 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 57 op/s; 11/627 objects degraded (1.754%); 0 B/s, 7 objects/s recovering 2024-02-07T12:54:47.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:46 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:54:47.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:46 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:54:48.270 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:54:48.270 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:54:48.271 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:48.271 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:48.271 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:48.271 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:48.271 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:48.271 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-02-07T12:54:48.271 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-02-07T12:54:48.271 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-02-07T12:54:48.271 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-02-07T12:54:48.271 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-02-07T12:54:48.271 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-02-07T12:54:48.271 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-02-07T12:54:48.271 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-02-07T12:54:48.271 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-02-07T12:54:48.271 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-02-07T12:54:48.271 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-02-07T12:54:48.272 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-02-07T12:54:48.272 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-02-07T12:54:48.272 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-02-07T12:54:48.272 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-02-07T12:54:48.272 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-02-07T12:54:48.272 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-02-07T12:54:48.272 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-02-07T12:54:48.272 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-02-07T12:54:48.272 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-02-07T12:54:48.272 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-02-07T12:54:48.272 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-02-07T12:54:48.272 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-02-07T12:54:48.272 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-02-07T12:54:48.272 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-02-07T12:54:48.272 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:47 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-02-07T12:54:48.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:54:48.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:54:48.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:48.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:48.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:48.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:48.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-02-07T12:54:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-02-07T12:54:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-02-07T12:54:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-02-07T12:54:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-02-07T12:54:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-02-07T12:54:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-02-07T12:54:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-02-07T12:54:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-02-07T12:54:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-02-07T12:54:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-02-07T12:54:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-02-07T12:54:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-02-07T12:54:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-02-07T12:54:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-02-07T12:54:48.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-02-07T12:54:48.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-02-07T12:54:48.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-02-07T12:54:48.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:54:48.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:54:48.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:48.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:48.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:48.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:48.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:48.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-02-07T12:54:48.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-02-07T12:54:48.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-02-07T12:54:48.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-02-07T12:54:48.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-02-07T12:54:48.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-02-07T12:54:48.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-02-07T12:54:48.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-02-07T12:54:48.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-02-07T12:54:48.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-02-07T12:54:48.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-02-07T12:54:48.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-02-07T12:54:48.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-02-07T12:54:48.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-02-07T12:54:48.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-02-07T12:54:48.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-02-07T12:54:48.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-02-07T12:54:48.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-02-07T12:54:48.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-02-07T12:54:48.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-02-07T12:54:48.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-02-07T12:54:48.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-02-07T12:54:48.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-02-07T12:54:48.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-02-07T12:54:48.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-02-07T12:54:48.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-02-07T12:54:48.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-02-07T12:54:48.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-02-07T12:54:48.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-02-07T12:54:48.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-02-07T12:54:48.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-02-07T12:54:48.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-02-07T12:54:48.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-02-07T12:54:48.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:47 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-02-07T12:54:49.237 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:49.237 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:54:49.237 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:49.237 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:49.237 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:49.237 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:49.237 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:49.237 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:54:49.237 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:49.237 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:54:49.237 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:49.237 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:54:49.238 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:49.238 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:49.238 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:49.238 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:49.238 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:49.238 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:49.238 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:49.238 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:49.238 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:49.238 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:54:49.238 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:54:49.238 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:54:49.238 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:54:49.238 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:54:49.238 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:54:49.239 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:54:49.239 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:54:49.239 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:54:49.239 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:54:49.239 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:54:49.239 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:54:49.239 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:54:49.239 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:54:49.239 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:54:49.239 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:54:49.239 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:54:49.239 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:49.239 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:54:49.239 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:49.239 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:49.240 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:49.240 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:49.240 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:49.240 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:54:49.240 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:49.240 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:54:49.240 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:49.240 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:54:49.240 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:49.240 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:49.240 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:49.240 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:49.241 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:49.241 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:54:49.241 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:49.241 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:49.241 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:49.241 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:49.241 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:49.241 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:54:49.242 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:49.242 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:54:49.242 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:49.242 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:54:49.242 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:49.242 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:49.242 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:49.242 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:49.242 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:49.242 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:49.242 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:49.242 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:49.242 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:49.242 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:54:49.242 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:54:49.243 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:54:49.243 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:54:49.243 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:54:49.243 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:54:49.243 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:54:49.243 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:54:49.243 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:54:49.243 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:54:49.243 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:54:49.243 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:54:49.243 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:54:49.243 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:54:49.243 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:54:49.243 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:54:49.244 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:54:49.244 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:49.244 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:54:49.244 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:49.244 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:49.244 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:49.244 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:49.244 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:49.244 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:54:49.244 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:49.244 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:54:49.244 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:49.244 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:54:49.244 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:49.245 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:49.245 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:49.245 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:49.245 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:49.245 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:49.245 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:49.245 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:49.245 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:49.245 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all osd 2024-02-07T12:54:49.245 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 45 op/s; 0 B/s, 8 objects/s recovering 2024-02-07T12:54:49.245 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: Upgrade: Setting require_osd_release to 19 squid 2024-02-07T12:54:49.246 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11/627 objects degraded (1.754%), 3 pgs degraded) 2024-02-07T12:54:49.246 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-02-07T12:54:49.246 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-02-07T12:54:49.246 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: osdmap e146: 8 total, 8 up, 8 in 2024-02-07T12:54:49.246 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.246 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:49.246 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.246 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:49.246 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.246 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.246 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:49.246 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.246 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:49.247 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.247 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:49.247 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.247 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.247 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.247 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.247 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.247 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.247 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:49.247 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:49.248 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:49.248 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:49.248 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:49.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all osd 2024-02-07T12:54:49.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 45 op/s; 0 B/s, 8 objects/s recovering 2024-02-07T12:54:49.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: Upgrade: Setting require_osd_release to 19 squid 2024-02-07T12:54:49.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11/627 objects degraded (1.754%), 3 pgs degraded) 2024-02-07T12:54:49.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-02-07T12:54:49.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-02-07T12:54:49.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: osdmap e146: 8 total, 8 up, 8 in 2024-02-07T12:54:49.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:49.248 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.249 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:49.249 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.249 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.249 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:49.249 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.249 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:49.249 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.249 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:49.249 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.249 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.249 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.249 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.249 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.249 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.250 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:54:49.250 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:54:49.250 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-02-07T12:54:49.250 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.250 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.250 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-02-07T12:54:49.250 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:54:49.250 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:54:49.250 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-02-07T12:54:49.250 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:54:49.250 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:54:49.251 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-02-07T12:54:49.251 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:54:49.251 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:54:49.251 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-02-07T12:54:49.251 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:54:49.251 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:54:49.251 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:54:49.251 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:54:49.251 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-02-07T12:54:49.251 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.251 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.251 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:54:49.251 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:54:49.252 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:54:49.252 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:54:49.252 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-02-07T12:54:49.252 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.252 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.252 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-02-07T12:54:49.252 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:54:49.252 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:54:49.253 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-02-07T12:54:49.253 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:54:49.253 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:54:49.253 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-02-07T12:54:49.253 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:54:49.253 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:54:49.253 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-02-07T12:54:49.253 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:54:49.253 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:54:49.253 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:54:49.253 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:54:49.253 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-02-07T12:54:49.253 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.253 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.253 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:54:49.254 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:54:49.254 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-02-07T12:54:49.254 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:54:49.254 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:54:49.254 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:54:49.254 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:54:49.254 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-02-07T12:54:49.254 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:54:49.254 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:54:49.254 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-02-07T12:54:49.254 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.254 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.254 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.254 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.254 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.254 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.255 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.255 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.255 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.255 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.255 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.255 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.255 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:54:49.255 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:54:49.255 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-02-07T12:54:49.255 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-02-07T12:54:49.256 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:54:49.256 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:54:49.256 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:54:49.256 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:54:49.256 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-02-07T12:54:49.256 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:54:49.256 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:54:49.256 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-02-07T12:54:49.256 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.256 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.256 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.256 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.256 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.256 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.257 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.257 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.257 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.257 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.257 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.257 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.257 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:54:49.257 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:54:49.257 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-02-07T12:54:49.257 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:54:49.257 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:54:49.257 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:54:49.257 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:49.257 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:54:49.257 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:54:49.258 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:54:49.258 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:54:49.258 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:54:49.258 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:54:49.258 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:49.258 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:54:49.258 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:54:49.258 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:48 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:54:49.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:49.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:54:49.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:49.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:49.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:49.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:49.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:49.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:54:49.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:49.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:54:49.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:49.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:54:49.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:49.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:49.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:49.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:49.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:49.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:49.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:49.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:49.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:49.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:54:49.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:54:49.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:54:49.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:54:49.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:54:49.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:54:49.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:54:49.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:54:49.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:54:49.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:54:49.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:54:49.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:54:49.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:54:49.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:54:49.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:54:49.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:54:49.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:54:49.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:49.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:54:49.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:49.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:49.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:49.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:49.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:49.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:54:49.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:49.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:54:49.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:49.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:54:49.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:49.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:49.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:49.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:49.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:49.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:49.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:49.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:49.374 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:49.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all osd 2024-02-07T12:54:49.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 45 op/s; 0 B/s, 8 objects/s recovering 2024-02-07T12:54:49.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: Upgrade: Setting require_osd_release to 19 squid 2024-02-07T12:54:49.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11/627 objects degraded (1.754%), 3 pgs degraded) 2024-02-07T12:54:49.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-02-07T12:54:49.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-02-07T12:54:49.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: osdmap e146: 8 total, 8 up, 8 in 2024-02-07T12:54:49.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:49.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:49.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:49.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:49.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:49.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:49.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:54:49.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:54:49.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-02-07T12:54:49.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-02-07T12:54:49.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:54:49.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:54:49.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-02-07T12:54:49.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:54:49.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:54:49.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-02-07T12:54:49.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:54:49.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:54:49.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-02-07T12:54:49.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:54:49.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:54:49.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:54:49.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:54:49.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-02-07T12:54:49.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.376 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:54:49.377 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:54:49.377 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-02-07T12:54:49.377 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:54:49.377 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:54:49.377 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:54:49.377 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:54:49.377 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-02-07T12:54:49.377 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:54:49.378 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:54:49.378 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-02-07T12:54:49.378 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.378 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.378 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.378 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.378 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.378 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.378 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.378 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.378 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.378 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.378 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.378 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:54:49.378 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:54:49.378 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:54:49.379 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-02-07T12:54:49.379 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:54:49.379 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:54:49.379 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:54:49.379 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:49.379 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:54:49.379 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:54:49.379 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:48 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:54:50.187 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:49 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all mds 2024-02-07T12:54:50.187 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:49 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all rbd-mirror 2024-02-07T12:54:50.187 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:49 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all ceph-exporter 2024-02-07T12:54:50.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:49 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all nfs 2024-02-07T12:54:50.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:49 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all nvmeof 2024-02-07T12:54:50.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:49 smithi019 ceph-mon[82918]: Upgrade: Finalizing container_image settings 2024-02-07T12:54:50.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:49 smithi019 ceph-mon[82918]: Upgrade: Complete! 2024-02-07T12:54:50.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:49 smithi019 ceph-mon[82918]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 6.2 KiB/s rd, 0 B/s wr, 9 op/s; 0 B/s, 8 objects/s recovering 2024-02-07T12:54:50.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:49 smithi019 ceph-mon[82918]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 0 B/s wr, 9 op/s; 0 B/s, 10 objects/s recovering 2024-02-07T12:54:50.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:49 smithi019 ceph-mon[82918]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:54:50.188 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:49 smithi019 ceph-mon[82918]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:54:50.188 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:49 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all mds 2024-02-07T12:54:50.188 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:49 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all rbd-mirror 2024-02-07T12:54:50.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:49 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all ceph-exporter 2024-02-07T12:54:50.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:49 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all nfs 2024-02-07T12:54:50.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:49 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all nvmeof 2024-02-07T12:54:50.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:49 smithi019 ceph-mon[88137]: Upgrade: Finalizing container_image settings 2024-02-07T12:54:50.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:49 smithi019 ceph-mon[88137]: Upgrade: Complete! 2024-02-07T12:54:50.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:49 smithi019 ceph-mon[88137]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 6.2 KiB/s rd, 0 B/s wr, 9 op/s; 0 B/s, 8 objects/s recovering 2024-02-07T12:54:50.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:49 smithi019 ceph-mon[88137]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 0 B/s wr, 9 op/s; 0 B/s, 10 objects/s recovering 2024-02-07T12:54:50.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:49 smithi019 ceph-mon[88137]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:54:50.189 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:49 smithi019 ceph-mon[88137]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:54:50.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:49 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all mds 2024-02-07T12:54:50.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:49 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all rbd-mirror 2024-02-07T12:54:50.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:49 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all ceph-exporter 2024-02-07T12:54:50.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:49 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all nfs 2024-02-07T12:54:50.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:49 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all nvmeof 2024-02-07T12:54:50.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:49 smithi172 ceph-mon[67636]: Upgrade: Finalizing container_image settings 2024-02-07T12:54:50.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:49 smithi172 ceph-mon[67636]: Upgrade: Complete! 2024-02-07T12:54:50.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:49 smithi172 ceph-mon[67636]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 6.2 KiB/s rd, 0 B/s wr, 9 op/s; 0 B/s, 8 objects/s recovering 2024-02-07T12:54:50.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:49 smithi172 ceph-mon[67636]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 0 B/s wr, 9 op/s; 0 B/s, 10 objects/s recovering 2024-02-07T12:54:50.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:49 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:54:50.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:49 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:54:50.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:54:50.184+0000 7fb71d671700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:50.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:54:50.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:50.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:50.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:50.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:50.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:50.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:54:50.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:50.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:54:50.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:50.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:54:50.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:50.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:50.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:50.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:50.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:50.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:50.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:50.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:50.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:50.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:54:50.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:54:50.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:54:50.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:54:50.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:54:50.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:54:50.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:54:50.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:54:50.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:54:50.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:54:50.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:54:50.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:54:50.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:54:50.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:54:50.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:54:50.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:54:50.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:54:50.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:50.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:54:50.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:50.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:50.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:50.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:50.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:50.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:54:50.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:50.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:54:50.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:50.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:54:50.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:50.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:50.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:50.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:50.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:50.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:50.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:50.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:50.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:50 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:51.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:51.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:54:51.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[82918]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:51.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:54:51.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:51.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:51.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:51.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:51.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:51.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:54:51.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:51.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:54:51.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:51.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:54:51.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:51.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:51.345 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:51.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:51.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:51.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:51.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:51.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:51.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:51.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:54:51.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:54:51.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:54:51.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:54:51.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:54:51.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:54:51.346 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:54:51.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:54:51.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:54:51.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:54:51.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:54:51.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:54:51.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:54:51.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:54:51.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:54:51.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:54:51.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:54:51.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:51.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:54:51.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:51.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:51.347 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:51.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:51.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:51.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:54:51.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:51.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:54:51.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:51.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:54:51.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:51.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:51.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:51.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:51.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:54:51.349 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[88137]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:51.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:54:51.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:51.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:51.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:51.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:51.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:51.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:54:51.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:51.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:54:51.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:51.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:54:51.349 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:51.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:51.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:51.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:51.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:51.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:51.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:51.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:51.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:51.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:54:51.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:54:51.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:54:51.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:54:51.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:54:51.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:54:51.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:54:51.350 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:54:51.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:54:51.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:54:51.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:54:51.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:54:51.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:54:51.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:54:51.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:54:51.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:54:51.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:54:51.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:51.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:54:51.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:51.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:51.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:51.351 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:51.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:51.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:54:51.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:51.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:54:51.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:51.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:54:51.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:51.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:51.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:51.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:51.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:51.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:51.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:51.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:51.353 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:51.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:54:51.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:54:51.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:54:51.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[82918]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s; 0 B/s, 2 objects/s recovering 2024-02-07T12:54:51.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[82918]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s; 0 B/s, 2 objects/s recovering 2024-02-07T12:54:51.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:51.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[82918]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:54:51.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:54:51.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:54:51.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:54:51.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[82918]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:54:51.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:51.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[82918]: from='client.54550 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:54:51.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:51.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:51.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:51.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:51.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:51.354 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:51.354 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:54:51.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:54:51.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:54:51.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[88137]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s; 0 B/s, 2 objects/s recovering 2024-02-07T12:54:51.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[88137]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s; 0 B/s, 2 objects/s recovering 2024-02-07T12:54:51.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:51.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[88137]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:54:51.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:54:51.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:54:51.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:54:51.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[88137]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:54:51.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:51.355 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:51 smithi019 ceph-mon[88137]: from='client.54550 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:54:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:51 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:51 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:54:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:51 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:54:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:51.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:51.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:51.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:51.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:54:51.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:51.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:54:51.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:51.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:54:51.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:51.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:51.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:51.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:51.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:51.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:51.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:51.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:51.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:51.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:54:51.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:54:51.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:54:51.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:54:51.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:54:51.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:54:51.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:54:51.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:54:51.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:54:51.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:54:51.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:54:51.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:54:51.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:54:51.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:54:51.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:54:51.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:54:51.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:54:51.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:51.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:54:51.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:51.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:51.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:51.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:51.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:51.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:54:51.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:51.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:54:51.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:51.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:54:51.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:51.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:51.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:51.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:51.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:51.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:51.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:51.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:51.374 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:51.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:51 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:54:51.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:51 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:54:51.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:51 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:54:51.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:51 smithi172 ceph-mon[67636]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s; 0 B/s, 2 objects/s recovering 2024-02-07T12:54:51.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:51 smithi172 ceph-mon[67636]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s; 0 B/s, 2 objects/s recovering 2024-02-07T12:54:51.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:51 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:51.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:51 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:54:51.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:51 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:54:51.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:51 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:54:51.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:51 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:54:51.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:51 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:54:51.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:51 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:51.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:51 smithi172 ceph-mon[67636]: from='client.54550 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:54:51.461 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch ps' 2024-02-07T12:54:53.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:54:52.905+0000 7fb71d671700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:53.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:54:53.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:53.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:53.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:53.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:53.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:53.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:54:53.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:53.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:54:53.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:53.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:54:53.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:53.095 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:53.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:53.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:53.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:53.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:53.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:53.096 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:53.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:53.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:54:53.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:54:53.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:54:53.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:54:53.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:54:53.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:54:53.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:54:53.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:54:53.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:54:53.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:54:53.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:54:53.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:54:53.097 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:54:53.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:54:53.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:54:53.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:54:53.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:54:53.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:53.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:54:53.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:53.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:53.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:53.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:53.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:53.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:54:53.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:53.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:54:53.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:53.098 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:54:53.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:53.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:53.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:53.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:53.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:53.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:53.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:53.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:53.099 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:52 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:53.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:53 smithi019 ceph-mon[82918]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s 2024-02-07T12:54:53.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:53 smithi019 ceph-mon[82918]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:53.594 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:54:53.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:53.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:53.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:53.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:53.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:53.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:54:53.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:53.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:54:53.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:53.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:54:53.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:53.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:53.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:53.595 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:53.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:53.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:53.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:53.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:53.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:53.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:54:53.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:54:53.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:54:53.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:54:53.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:54:53.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:54:53.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:54:53.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:54:53.596 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:54:53.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:54:53.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:54:53.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:54:53.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:54:53.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:54:53.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:54:53.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:54:53.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:54:53.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:53.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:54:53.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:53.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:53.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:53.597 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:53.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:53.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:54:53.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:53.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:54:53.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:53.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:54:53.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:53.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:53.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:53.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:53.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:53.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:53.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:53.598 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:53.599 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:53.599 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:53 smithi019 ceph-mon[88137]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s 2024-02-07T12:54:53.599 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:53 smithi019 ceph-mon[88137]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:53.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:54:53.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:53.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:53.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:53.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:53.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:53.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:54:53.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:53.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:54:53.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:53.599 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:54:53.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:53.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:53.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:53.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:53.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:53.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:53.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:53.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:53.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:53.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:54:53.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:54:53.600 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:54:53.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:54:53.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:54:53.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:54:53.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:54:53.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:54:53.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:54:53.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:54:53.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:54:53.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:54:53.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:54:53.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:54:53.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:54:53.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:54:53.601 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:54:53.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:53.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:54:53.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:53.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:53.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:53.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:53.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:53.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:54:53.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:53.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:54:53.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:53.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:54:53.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:53.602 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:53.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:53.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:53.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:53.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:53.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:53.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:53.603 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:53.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:53 smithi172 ceph-mon[67636]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s 2024-02-07T12:54:53.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:53 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:53.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:54:53.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:53.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:53.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:53.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:53.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:53.619 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:54:53.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:53.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:54:53.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:53.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:54:53.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:53.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:53.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:53.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:53.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:53.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:53.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:53.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:53.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:53.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:54:53.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:54:53.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:54:53.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:54:53.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:54:53.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:54:53.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:54:53.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:54:53.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:54:53.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:54:53.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:54:53.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:54:53.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:54:53.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:54:53.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:54:53.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:54:53.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:54:53.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:54:53.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:54:53.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:54:53.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:54:53.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:54:53.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:53.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:53.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:54:53.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:54:53.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:54:53.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:54:53.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:54:53.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:54:53.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:54:53.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:54:53.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:54:53.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:54:53.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:54:53.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:54:53.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:54:53.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:54:53.964 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:54:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:54:53] "GET /metrics HTTP/1.1" 200 38380 "" "Prometheus/2.43.0" 2024-02-07T12:54:54.399 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:54:54.399 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (9m) 2m ago 30m 27.3M - 0.25.0 c8568f914cd2 fec9255f35e7 2024-02-07T12:54:54.399 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (8m) 14s ago 30m 85.7M - 9.4.12 a0ea5e3c46d9 aaffac63bc28 2024-02-07T12:54:54.400 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 unknown 2m ago 30m 63.5M - 3.5 e1d6a67b021e 5113aa77e3f2 2024-02-07T12:54:54.400 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283,8765 running (10m) 14s ago 35m 452M - 19.0.0-1252-g44458cce 79957b372307 5aa0c04fdfd7 2024-02-07T12:54:54.400 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (20m) 2m ago 37m 509M - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:54:54.400 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (6m) 2m ago 37m 53.6M 2048M 19.0.0-1252-g44458cce 79957b372307 8b1b0e21b1c5 2024-02-07T12:54:54.400 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (7m) 14s ago 35m 52.9M 2048M 19.0.0-1252-g44458cce 79957b372307 00f454bbac01 2024-02-07T12:54:54.400 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (6m) 2m ago 35m 43.3M 2048M 19.0.0-1252-g44458cce 79957b372307 6214003f6e08 2024-02-07T12:54:54.400 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (9m) 2m ago 31m 16.0M - 1.5.0 0da6a335fe13 c4bdcb1c4d49 2024-02-07T12:54:54.401 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (9m) 14s ago 31m 16.6M - 1.5.0 0da6a335fe13 04ea81f79d85 2024-02-07T12:54:54.401 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (4m) 2m ago 35m 56.0M 2993M 19.0.0-1252-g44458cce 79957b372307 78eddd34a903 2024-02-07T12:54:54.401 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (4m) 2m ago 34m 44.0M 2993M 19.0.0-1252-g44458cce 79957b372307 c37447e76c38 2024-02-07T12:54:54.401 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (3m) 2m ago 34m 50.2M 2993M 19.0.0-1252-g44458cce 79957b372307 90352897da85 2024-02-07T12:54:54.401 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (2m) 2m ago 34m 14.9M 2993M 19.0.0-1252-g44458cce 79957b372307 067cd6b3fd2f 2024-02-07T12:54:54.401 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (2m) 14s ago 33m 47.3M 3248M 19.0.0-1252-g44458cce 79957b372307 dbb47a8ad5ac 2024-02-07T12:54:54.401 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (91s) 14s ago 33m 45.3M 3248M 19.0.0-1252-g44458cce 79957b372307 d139d35fec57 2024-02-07T12:54:54.401 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (55s) 14s ago 32m 51.7M 3248M 19.0.0-1252-g44458cce 79957b372307 219cf25240d5 2024-02-07T12:54:54.401 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (18s) 14s ago 32m 15.0M 3248M 19.0.0-1252-g44458cce 79957b372307 75dab2b6606b 2024-02-07T12:54:54.401 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (9m) 14s ago 31m 71.6M - 2.43.0 a07b618ecd1d c9b607eea1be 2024-02-07T12:54:54.401 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (30m) 2m ago 30m 80.2M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:54:54.402 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (30m) 14s ago 30m 80.5M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:54:55.060 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 1'"'"'' 2024-02-07T12:54:56.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:55 smithi019 ceph-mon[82918]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 843 B/s rd, 0 op/s 2024-02-07T12:54:56.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:55 smithi019 ceph-mon[82918]: from='client.44616 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:54:56.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:55 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:56.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:55 smithi019 ceph-mon[88137]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 843 B/s rd, 0 op/s 2024-02-07T12:54:56.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:55 smithi019 ceph-mon[88137]: from='client.44616 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:54:56.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:55 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:56.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:55 smithi172 ceph-mon[67636]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 843 B/s rd, 0 op/s 2024-02-07T12:54:56.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:55 smithi172 ceph-mon[67636]: from='client.44616 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:54:56.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:55 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:54:57.289 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:54:57.900 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph versions | jq -e '"'"'.osd | keys'"'"' | grep $sha1' 2024-02-07T12:54:58.215 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:57 smithi019 ceph-mon[88137]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 646 B/s rd, 0 op/s 2024-02-07T12:54:58.215 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:57 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:54:58.215 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:57 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/1024872062' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:58.215 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:57 smithi019 ceph-mon[82918]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 646 B/s rd, 0 op/s 2024-02-07T12:54:58.215 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:57 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:54:58.215 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:57 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/1024872062' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:54:58.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:57 smithi172 ceph-mon[67636]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 646 B/s rd, 0 op/s 2024-02-07T12:54:58.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:57 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:54:58.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:57 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/1024872062' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:00.041 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)" 2024-02-07T12:55:00.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:54:59 smithi019 ceph-mon[82918]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:55:00.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:54:59 smithi019 ceph-mon[88137]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:55:00.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:54:59 smithi172 ceph-mon[67636]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:55:00.690 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch upgrade status' 2024-02-07T12:55:01.244 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:00 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/466259946' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:01.244 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:00 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/466259946' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:01.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:00 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/466259946' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:01.990 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:01 smithi019 ceph-mon[82918]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:55:01.990 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:01 smithi019 ceph-mon[88137]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:55:02.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:01 smithi172 ceph-mon[67636]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:55:02.737 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:55:02.737 INFO:teuthology.orchestra.run.smithi019.stdout: "target_image": null, 2024-02-07T12:55:02.737 INFO:teuthology.orchestra.run.smithi019.stdout: "in_progress": false, 2024-02-07T12:55:02.737 INFO:teuthology.orchestra.run.smithi019.stdout: "which": "", 2024-02-07T12:55:02.737 INFO:teuthology.orchestra.run.smithi019.stdout: "services_complete": [], 2024-02-07T12:55:02.737 INFO:teuthology.orchestra.run.smithi019.stdout: "progress": null, 2024-02-07T12:55:02.737 INFO:teuthology.orchestra.run.smithi019.stdout: "message": "", 2024-02-07T12:55:02.737 INFO:teuthology.orchestra.run.smithi019.stdout: "is_paused": false 2024-02-07T12:55:02.737 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:55:03.341 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph health detail' 2024-02-07T12:55:04.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:03 smithi019 ceph-mon[82918]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:55:04.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:03 smithi019 ceph-mon[82918]: from='client.44634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:55:04.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:03 smithi019 ceph-mon[88137]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:55:04.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:03 smithi019 ceph-mon[88137]: from='client.44634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:55:04.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:55:03] "GET /metrics HTTP/1.1" 200 38380 "" "Prometheus/2.43.0" 2024-02-07T12:55:04.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:03 smithi172 ceph-mon[67636]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:55:04.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:03 smithi172 ceph-mon[67636]: from='client.44634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:55:05.289 INFO:teuthology.orchestra.run.smithi019.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-02-07T12:55:05.289 INFO:teuthology.orchestra.run.smithi019.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-02-07T12:55:05.289 INFO:teuthology.orchestra.run.smithi019.stdout: daemon iscsi.foo.smithi019.svjimj on smithi019 is in unknown state 2024-02-07T12:55:05.911 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo' 2024-02-07T12:55:06.226 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:05 smithi019 ceph-mon[82918]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:55:06.226 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:55:06.226 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:05 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/1535999717' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:55:06.226 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:05 smithi019 ceph-mon[88137]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:55:06.227 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:55:06.227 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:05 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/1535999717' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:55:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:05 smithi172 ceph-mon[67636]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:55:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:55:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:05 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/1535999717' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:55:08.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:07 smithi019 ceph-mon[88137]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:55:08.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:07 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:55:08.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:07 smithi019 ceph-mon[82918]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:55:08.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:07 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:55:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:07 smithi172 ceph-mon[67636]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:55:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:07 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:55:09.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:08 smithi019 ceph-mon[88137]: from='client.44646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:55:09.104 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:08 smithi019 ceph-mon[82918]: from='client.44646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:55:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:08 smithi172 ceph-mon[67636]: from='client.44646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:55:10.120 INFO:teuthology.orchestra.run.smithi019.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:55:10.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:09 smithi019 ceph-mon[88137]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:55:10.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:09 smithi019 ceph-mon[82918]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:55:10.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:09 smithi172 ceph-mon[67636]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:55:10.767 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-02-07T12:55:11.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:11 smithi019 ceph-mon[88137]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:55:11.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:11 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:11.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:11 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:55:11.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:11 smithi019 ceph-mon[88137]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:55:11.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:11 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:11.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:11 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:55:11.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:11 smithi019 ceph-mon[88137]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 954 B/s rd, 0 op/s 2024-02-07T12:55:11.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:11 smithi019 ceph-mon[88137]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:55:11.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:11 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:11.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:11 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:55:11.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:11 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:55:11.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:11 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:11 smithi019 ceph-mon[82918]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:55:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:11 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:11 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:55:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:11 smithi019 ceph-mon[82918]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:55:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:11 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:11 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:55:11.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:11 smithi019 ceph-mon[82918]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 954 B/s rd, 0 op/s 2024-02-07T12:55:11.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:11 smithi019 ceph-mon[82918]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:55:11.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:11 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:11.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:11 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:55:11.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:11 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:55:11.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:11 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:11.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:11 smithi172 ceph-mon[67636]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:55:11.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:11 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:11.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:11 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:55:11.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:11 smithi172 ceph-mon[67636]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:55:11.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:11 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:11 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:55:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:11 smithi172 ceph-mon[67636]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 954 B/s rd, 0 op/s 2024-02-07T12:55:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:11 smithi172 ceph-mon[67636]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:55:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:11 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:11 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:55:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:11 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:55:11.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:11 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:12.281 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:12 smithi019 ceph-mon[88137]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:55:12.281 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:12 smithi019 ceph-mon[88137]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:55:12.281 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:12 smithi019 ceph-mon[82918]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:55:12.281 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:12 smithi019 ceph-mon[82918]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:55:12.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:12 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:55:12.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:12 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:55:12.739 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:55:13.329 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-mon[88137]: from='client.44652 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:55:13.330 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-mon[88137]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 586 B/s rd, 0 op/s 2024-02-07T12:55:13.330 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-mon[82918]: from='client.44652 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:55:13.330 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-mon[82918]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 586 B/s rd, 0 op/s 2024-02-07T12:55:13.418 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:55:13.418 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (9m) 2m ago 31m 27.3M - 0.25.0 c8568f914cd2 fec9255f35e7 2024-02-07T12:55:13.418 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (9m) 33s ago 31m 85.7M - 9.4.12 a0ea5e3c46d9 aaffac63bc28 2024-02-07T12:55:13.418 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 unknown 2m ago 30m 63.5M - 3.5 e1d6a67b021e 5113aa77e3f2 2024-02-07T12:55:13.419 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283,8765 running (10m) 33s ago 35m 452M - 19.0.0-1252-g44458cce 79957b372307 5aa0c04fdfd7 2024-02-07T12:55:13.419 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (20m) 2m ago 37m 509M - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:55:13.419 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (7m) 2m ago 37m 53.6M 2048M 19.0.0-1252-g44458cce 79957b372307 8b1b0e21b1c5 2024-02-07T12:55:13.419 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (8m) 33s ago 36m 52.9M 2048M 19.0.0-1252-g44458cce 79957b372307 00f454bbac01 2024-02-07T12:55:13.419 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (6m) 2m ago 36m 43.3M 2048M 19.0.0-1252-g44458cce 79957b372307 6214003f6e08 2024-02-07T12:55:13.419 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (10m) 2m ago 32m 16.0M - 1.5.0 0da6a335fe13 c4bdcb1c4d49 2024-02-07T12:55:13.419 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (10m) 33s ago 31m 16.6M - 1.5.0 0da6a335fe13 04ea81f79d85 2024-02-07T12:55:13.419 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (5m) 2m ago 35m 56.0M 2993M 19.0.0-1252-g44458cce 79957b372307 78eddd34a903 2024-02-07T12:55:13.419 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (4m) 2m ago 35m 44.0M 2993M 19.0.0-1252-g44458cce 79957b372307 c37447e76c38 2024-02-07T12:55:13.419 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (3m) 2m ago 34m 50.2M 2993M 19.0.0-1252-g44458cce 79957b372307 90352897da85 2024-02-07T12:55:13.419 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (2m) 2m ago 34m 14.9M 2993M 19.0.0-1252-g44458cce 79957b372307 067cd6b3fd2f 2024-02-07T12:55:13.419 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (2m) 33s ago 34m 47.3M 3248M 19.0.0-1252-g44458cce 79957b372307 dbb47a8ad5ac 2024-02-07T12:55:13.419 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (110s) 33s ago 33m 45.3M 3248M 19.0.0-1252-g44458cce 79957b372307 d139d35fec57 2024-02-07T12:55:13.419 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (74s) 33s ago 33m 51.7M 3248M 19.0.0-1252-g44458cce 79957b372307 219cf25240d5 2024-02-07T12:55:13.419 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (37s) 33s ago 32m 15.0M 3248M 19.0.0-1252-g44458cce 79957b372307 75dab2b6606b 2024-02-07T12:55:13.420 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (9m) 33s ago 32m 71.6M - 2.43.0 a07b618ecd1d c9b607eea1be 2024-02-07T12:55:13.420 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (30m) 2m ago 30m 80.2M - 17.2.0 e1d6a67b021e 9272719eb7b6 2024-02-07T12:55:13.420 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (30m) 33s ago 30m 80.5M - 17.2.0 e1d6a67b021e 174f0bd68438 2024-02-07T12:55:13.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:13 smithi172 ceph-mon[67636]: from='client.44652 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:55:13.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:13 smithi172 ceph-mon[67636]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 586 B/s rd, 0 op/s 2024-02-07T12:55:13.831 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:55:13.832 INFO:teuthology.orchestra.run.smithi019.stdout: "mon": { 2024-02-07T12:55:13.832 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 3 2024-02-07T12:55:13.832 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:55:13.832 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr": { 2024-02-07T12:55:13.832 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:55:13.832 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:55:13.832 INFO:teuthology.orchestra.run.smithi019.stdout: "osd": { 2024-02-07T12:55:13.832 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 8 2024-02-07T12:55:13.832 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:55:13.832 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw": { 2024-02-07T12:55:13.832 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:55:13.832 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:55:13.833 INFO:teuthology.orchestra.run.smithi019.stdout: "overall": { 2024-02-07T12:55:13.833 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-02-07T12:55:13.833 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 13 2024-02-07T12:55:13.833 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:55:13.833 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:55:13.915 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:55:13.834+0000 7fb71d671700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:13.915 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:55:13.915 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:13.916 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:13.916 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:13.916 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:13.916 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:13.916 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:55:13.916 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:13.916 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:55:13.916 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:13.916 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:55:13.916 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:13.916 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:13.916 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:13.916 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:13.917 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:13.917 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:13.917 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:13.917 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:13.917 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:13.917 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:55:13.917 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:55:13.917 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:55:13.918 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:55:13.918 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:55:13.918 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:55:13.918 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:55:13.918 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:55:13.918 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:55:13.918 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:55:13.918 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:55:13.918 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:55:13.918 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:55:13.918 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:55:13.918 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:55:13.918 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:55:13.919 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:55:13.919 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:13.919 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:55:13.919 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:13.919 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:13.919 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:13.919 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:13.919 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:13.919 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:55:13.919 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:13.919 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:55:13.919 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:13.919 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:55:13.919 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:13.919 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:13.920 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:13.920 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:13.920 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:13.920 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:13.920 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:13.920 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:13.920 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:13.920 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:55:13] "GET /metrics HTTP/1.1" 200 38370 "" "Prometheus/2.43.0" 2024-02-07T12:55:14.178 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:55:14.178 INFO:teuthology.orchestra.run.smithi019.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", 2024-02-07T12:55:14.178 INFO:teuthology.orchestra.run.smithi019.stdout: "in_progress": true, 2024-02-07T12:55:14.178 INFO:teuthology.orchestra.run.smithi019.stdout: "which": "Upgrading daemons in service(s) rgw.foo", 2024-02-07T12:55:14.179 INFO:teuthology.orchestra.run.smithi019.stdout: "services_complete": [], 2024-02-07T12:55:14.179 INFO:teuthology.orchestra.run.smithi019.stdout: "progress": "", 2024-02-07T12:55:14.179 INFO:teuthology.orchestra.run.smithi019.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 image", 2024-02-07T12:55:14.179 INFO:teuthology.orchestra.run.smithi019.stdout: "is_paused": false 2024-02-07T12:55:14.179 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:55:14.179 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:14 smithi019 ceph-mon[82918]: from='client.54598 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:55:14.179 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:14 smithi019 ceph-mon[82918]: from='client.44664 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:55:14.179 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:14 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/1580197396' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:14.179 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:14 smithi019 ceph-mon[82918]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:14.179 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:55:14.180 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:14.180 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:14.180 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:14.180 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:14.180 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:14.180 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:55:14.180 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:14.180 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:55:14.180 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:14.180 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:55:14.180 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:14.180 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:14.180 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:14.180 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:14.180 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:14.181 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:14.181 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:14.181 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:14.181 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:14 smithi019 ceph-mon[88137]: from='client.54598 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:55:14.181 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:14 smithi019 ceph-mon[88137]: from='client.44664 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:55:14.181 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:14 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/1580197396' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:14.181 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:14 smithi019 ceph-mon[88137]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:14.181 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:55:14.181 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:14.181 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:14.181 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:14.182 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:14.182 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:14.182 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:55:14.182 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:14.182 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:55:14.182 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:14.182 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:55:14.182 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:14.182 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:14.182 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:14.182 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:14.182 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:14.182 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:14.182 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:14.182 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:14.182 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:14.182 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:55:14.183 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:55:14.183 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:55:14.183 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:55:14.183 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:55:14.183 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:55:14.183 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:55:14.183 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:55:14.183 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:55:14.183 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:55:14.183 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:55:14.183 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:55:14.183 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:55:14.183 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:55:14.183 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:55:14.183 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:55:14.184 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:55:14.184 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:14.184 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:55:14.184 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:14.184 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:14.184 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:14.184 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:14.184 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:14.184 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:55:14.184 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:14.184 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:55:14.184 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:14.184 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:55:14.184 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:14.185 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:14.185 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:14.185 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:55:14.185 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:55:14.185 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:55:14.185 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:55:14.185 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:55:14.185 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:55:14.185 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:55:14.185 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:55:14.185 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:55:14.186 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:55:14.186 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:55:14.186 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:55:14.186 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:55:14.186 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:55:14.186 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:55:14.186 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:55:14.186 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:55:14.186 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:14.186 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:55:14.186 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:14.186 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:14.186 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:14.186 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:14.186 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:14.186 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:55:14.187 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:14.187 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:55:14.187 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:14.187 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:55:14.187 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:14.187 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:14.187 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:14.187 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:14.187 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:14.187 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:14.187 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:14.187 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:14.187 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:14.187 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:14 smithi019 ceph-mon[82918]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:55:14.187 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:14.188 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:14.188 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:14.188 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:14.188 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:14.188 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:14.188 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:14.188 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:14 smithi019 ceph-mon[88137]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:55:14.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:14 smithi172 ceph-mon[67636]: from='client.54598 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:55:14.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:14 smithi172 ceph-mon[67636]: from='client.44664 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:55:14.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:14 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/1580197396' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:14.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:14 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:14.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:55:14.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:14.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:14.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:14.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:14.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:14.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:55:14.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:14.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:55:14.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:14.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:55:14.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:14.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:55:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:55:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:55:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:55:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:55:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:55:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:55:14.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:55:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:55:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:55:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:55:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:55:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:55:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:55:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:55:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:55:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:55:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:55:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:14.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:55:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:55:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:55:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:14.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:14.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:14.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:14.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:14 smithi172 ceph-mon[67636]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:55:16.335 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:15 smithi019 ceph-mon[88137]: from='client.44676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:55:16.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:15 smithi019 ceph-mon[88137]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:55:16.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:15 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:16.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:15 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:55:16.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:15 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:16.336 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:15 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:16.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:15 smithi019 ceph-mon[82918]: from='client.44676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:55:16.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:15 smithi019 ceph-mon[82918]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:55:16.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:15 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:16.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:15 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:55:16.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:15 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:16.336 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:15 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:16.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:15 smithi172 ceph-mon[67636]: from='client.44676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:55:16.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:15 smithi172 ceph-mon[67636]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:55:16.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:15 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:16.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:15 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:55:16.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:15 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:16.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:15 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:17.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[88137]: Upgrade: Target is version 19.0.0-1252-g44458cce (squid) 2024-02-07T12:55:17.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[88137]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe'] 2024-02-07T12:55:17.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all mgr 2024-02-07T12:55:17.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:17.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all mon 2024-02-07T12:55:17.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:17.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:17.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all crash 2024-02-07T12:55:17.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:17.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:17.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all osd 2024-02-07T12:55:17.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:17.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:17.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all mds 2024-02-07T12:55:17.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:17.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[82918]: Upgrade: Target is version 19.0.0-1252-g44458cce (squid) 2024-02-07T12:55:17.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[82918]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe'] 2024-02-07T12:55:17.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all mgr 2024-02-07T12:55:17.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:17.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all mon 2024-02-07T12:55:17.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:17.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:17.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all crash 2024-02-07T12:55:17.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:17.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:17.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all osd 2024-02-07T12:55:17.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:17.096 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:17.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all mds 2024-02-07T12:55:17.097 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:16 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:17.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:16 smithi172 ceph-mon[67636]: Upgrade: Target is version 19.0.0-1252-g44458cce (squid) 2024-02-07T12:55:17.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:16 smithi172 ceph-mon[67636]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe'] 2024-02-07T12:55:17.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:16 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all mgr 2024-02-07T12:55:17.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:16 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:17.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:16 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all mon 2024-02-07T12:55:17.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:16 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:17.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:16 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:17.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:16 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all crash 2024-02-07T12:55:17.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:16 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:17.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:16 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:17.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:16 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all osd 2024-02-07T12:55:17.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:16 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:17.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:16 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:17.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:16 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all mds 2024-02-07T12:55:17.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:16 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:18.270 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:17 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:55:18.271 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:17 smithi172 ceph-mon[67636]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 586 B/s rd, 0 op/s 2024-02-07T12:55:18.271 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:17 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:18.271 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:17 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi019.ciualv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:55:18.271 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:17 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi019.ciualv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:55:18.271 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:17 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:18.273 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:17 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:55:18.273 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:17 smithi019 ceph-mon[88137]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 586 B/s rd, 0 op/s 2024-02-07T12:55:18.273 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:17 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:18.274 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:17 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi019.ciualv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:55:18.274 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:17 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi019.ciualv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:55:18.274 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:17 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:18.274 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:17 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:55:18.274 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:17 smithi019 ceph-mon[82918]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 586 B/s rd, 0 op/s 2024-02-07T12:55:18.274 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:17 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:18.274 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:17 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi019.ciualv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:55:18.274 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:17 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi019.ciualv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:55:18.274 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:17 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:19.262 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:18 smithi019 ceph-mon[88137]: Upgrade: Updating rgw.foo.smithi019.ciualv (1/2) 2024-02-07T12:55:19.263 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:18 smithi019 ceph-mon[88137]: Deploying daemon rgw.foo.smithi019.ciualv on smithi019 2024-02-07T12:55:19.263 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:18 smithi019 ceph-mon[82918]: Upgrade: Updating rgw.foo.smithi019.ciualv (1/2) 2024-02-07T12:55:19.263 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:18 smithi019 ceph-mon[82918]: Deploying daemon rgw.foo.smithi019.ciualv on smithi019 2024-02-07T12:55:19.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:18 smithi172 ceph-mon[67636]: Upgrade: Updating rgw.foo.smithi019.ciualv (1/2) 2024-02-07T12:55:19.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:18 smithi172 ceph-mon[67636]: Deploying daemon rgw.foo.smithi019.ciualv on smithi019 2024-02-07T12:55:20.167 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:19 smithi019 ceph-mon[82918]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:55:20.167 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:19 smithi019 ceph-mon[88137]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:55:20.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:19 smithi172 ceph-mon[67636]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:55:21.228 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:20 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:55:21.228 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:20 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:55:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:20 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:55:22.248 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:21 smithi172 ceph-mon[67636]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:55:22.249 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:21 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:22.249 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:21 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:22.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:21 smithi019 ceph-mon[82918]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:55:22.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:21 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:22.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:21 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:22.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:21 smithi019 ceph-mon[88137]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:55:22.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:21 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:22.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:21 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:24.074 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:23 smithi172 ceph-mon[67636]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 256 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 85 B/s wr, 27 op/s 2024-02-07T12:55:24.074 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:23 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:24.074 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:23 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.wueguy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:55:24.074 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:23 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.wueguy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:55:24.074 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:23 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:24.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:55:23] "GET /metrics HTTP/1.1" 200 38378 "" "Prometheus/2.43.0" 2024-02-07T12:55:24.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:23 smithi019 ceph-mon[82918]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 256 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 85 B/s wr, 27 op/s 2024-02-07T12:55:24.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:23 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:24.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:23 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.wueguy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:55:24.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:23 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.wueguy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:55:24.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:23 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:24.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:23 smithi019 ceph-mon[88137]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 256 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 85 B/s wr, 27 op/s 2024-02-07T12:55:24.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:23 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:24.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:23 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.wueguy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:55:24.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:23 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.wueguy", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:55:24.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:23 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:25.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:24 smithi172 ceph-mon[67636]: Upgrade: Updating rgw.foo.smithi172.wueguy (2/2) 2024-02-07T12:55:25.119 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:24 smithi172 ceph-mon[67636]: Deploying daemon rgw.foo.smithi172.wueguy on smithi172 2024-02-07T12:55:25.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:24 smithi019 ceph-mon[88137]: Upgrade: Updating rgw.foo.smithi172.wueguy (2/2) 2024-02-07T12:55:25.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:24 smithi019 ceph-mon[88137]: Deploying daemon rgw.foo.smithi172.wueguy on smithi172 2024-02-07T12:55:25.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:24 smithi019 ceph-mon[82918]: Upgrade: Updating rgw.foo.smithi172.wueguy (2/2) 2024-02-07T12:55:25.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:24 smithi019 ceph-mon[82918]: Deploying daemon rgw.foo.smithi172.wueguy on smithi172 2024-02-07T12:55:26.258 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:25 smithi172 ceph-mon[67636]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 256 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 85 B/s wr, 46 op/s 2024-02-07T12:55:26.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:25 smithi019 ceph-mon[82918]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 256 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 85 B/s wr, 46 op/s 2024-02-07T12:55:26.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:25 smithi019 ceph-mon[88137]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 256 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 85 B/s wr, 46 op/s 2024-02-07T12:55:28.266 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:27 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:55:28.266 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:27 smithi172 ceph-mon[67636]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 256 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 85 B/s wr, 84 op/s 2024-02-07T12:55:28.267 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:27 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:28.267 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:27 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:28.267 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:27 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:55:28.268 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:27 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:55:28.268 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:27 smithi019 ceph-mon[88137]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 256 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 85 B/s wr, 84 op/s 2024-02-07T12:55:28.268 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:27 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:28.269 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:27 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:28.269 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:27 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:55:28.269 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:27 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:55:28.269 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:27 smithi019 ceph-mon[82918]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 256 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 85 B/s wr, 84 op/s 2024-02-07T12:55:28.269 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:27 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:28.269 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:27 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:28.269 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:27 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:55:30.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:30 smithi019 ceph-mon[82918]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 76 KiB/s rd, 170 B/s wr, 116 op/s 2024-02-07T12:55:30.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:30 smithi019 ceph-mon[88137]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 76 KiB/s rd, 170 B/s wr, 116 op/s 2024-02-07T12:55:30.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:30 smithi172 ceph-mon[67636]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 76 KiB/s rd, 170 B/s wr, 116 op/s 2024-02-07T12:55:32.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:32 smithi019 ceph-mon[82918]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s rd, 170 B/s wr, 142 op/s 2024-02-07T12:55:32.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:32 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:32.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:32 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:32.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:32 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:32.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:32 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:32.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:32 smithi019 ceph-mon[88137]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s rd, 170 B/s wr, 142 op/s 2024-02-07T12:55:32.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:32 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:32.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:32 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:32.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:32 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:32.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:32 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:32.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:32 smithi172 ceph-mon[67636]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s rd, 170 B/s wr, 142 op/s 2024-02-07T12:55:32.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:32 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:32.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:32 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:32.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:32 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:32.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:32 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:33.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:33 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:33.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:33 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:33.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:33 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:33.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:33 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:33.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:33 smithi019 ceph-mon[82918]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 168 op/s 2024-02-07T12:55:33.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:33 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:33.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:33 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:33.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:33 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:33.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:33 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:33.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:33 smithi019 ceph-mon[88137]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 168 op/s 2024-02-07T12:55:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:33 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:33 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:33 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:33 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:33 smithi172 ceph-mon[67636]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 168 op/s 2024-02-07T12:55:34.027 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:55:33] "GET /metrics HTTP/1.1" 200 38378 "" "Prometheus/2.43.0" 2024-02-07T12:55:36.270 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:35 smithi019 ceph-mon[88137]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 85 B/s wr, 142 op/s 2024-02-07T12:55:36.270 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:35 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:55:36.270 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:35 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:36.270 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:35 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:36.270 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:35 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:36.270 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:35 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:36.270 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:35 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:36.270 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:35 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:55:36.270 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:35 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:36.271 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:35 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:55:36.271 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:35 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:55:36.271 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:35 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:36.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:35 smithi019 ceph-mon[82918]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 85 B/s wr, 142 op/s 2024-02-07T12:55:36.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:35 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:55:36.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:35 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:36.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:35 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:36.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:35 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:36.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:35 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:36.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:35 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:36.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:35 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:55:36.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:35 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:36.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:35 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:55:36.272 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:35 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:55:36.272 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:35 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:35 smithi172 ceph-mon[67636]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 85 B/s wr, 142 op/s 2024-02-07T12:55:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:35 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:55:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:35 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:35 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:35 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:36.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:35 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:36.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:35 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:36.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:35 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:55:36.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:35 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:36.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:35 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:55:36.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:35 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:55:36.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:35 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:37.203 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:36 smithi019 ceph-mon[82918]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 96 B/s wr, 139 op/s 2024-02-07T12:55:37.203 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:36 smithi019 ceph-mon[82918]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s rd, 119 B/s wr, 118 op/s 2024-02-07T12:55:37.203 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:36 smithi019 ceph-mon[82918]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:55:37.203 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:36 smithi019 ceph-mon[82918]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:55:37.203 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:36 smithi019 ceph-mon[88137]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 96 B/s wr, 139 op/s 2024-02-07T12:55:37.204 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:36 smithi019 ceph-mon[88137]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s rd, 119 B/s wr, 118 op/s 2024-02-07T12:55:37.204 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:36 smithi019 ceph-mon[88137]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:55:37.204 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:36 smithi019 ceph-mon[88137]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:55:37.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:36 smithi172 ceph-mon[67636]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 96 B/s wr, 139 op/s 2024-02-07T12:55:37.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:36 smithi172 ceph-mon[67636]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s rd, 119 B/s wr, 118 op/s 2024-02-07T12:55:37.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:36 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:55:37.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:36 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:55:37.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:55:37.331+0000 7fb71d671700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:37.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:55:37.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:37.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:37.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:37.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:37.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:37.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:55:37.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:37.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:55:37.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:37.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:55:37.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:37.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:37.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:37.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:37.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:37.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:37.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:37.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:37.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:37.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:55:37.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:55:37.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:55:37.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:55:37.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:55:37.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:55:37.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:55:37.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:55:37.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:55:37.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:55:37.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:55:37.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:55:37.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:55:37.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:55:37.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:55:37.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:55:37.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:55:37.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:37.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:55:37.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:37.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:37.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:37.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:37.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:37.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:55:37.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:37.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:55:37.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:37.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:55:37.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:37.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:37.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:37.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:37.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:37.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:37.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:37.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:37.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:38.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:55:38.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:55:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi019.ciualv"}]: dispatch 2024-02-07T12:55:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi019.ciualv"}]: dispatch 2024-02-07T12:55:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi019.ciualv"}]': finished 2024-02-07T12:55:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi172.wueguy"}]: dispatch 2024-02-07T12:55:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi172.wueguy"}]: dispatch 2024-02-07T12:55:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi172.wueguy"}]': finished 2024-02-07T12:55:38.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:55:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:55:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-02-07T12:55:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-02-07T12:55:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:55:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:55:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-02-07T12:55:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:55:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:55:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-02-07T12:55:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:55:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:55:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-02-07T12:55:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:55:38.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:55:38.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-02-07T12:55:38.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:55:38.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:55:38.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-02-07T12:55:38.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.348 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:55:38.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:55:38.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi019.ciualv"}]: dispatch 2024-02-07T12:55:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi019.ciualv"}]: dispatch 2024-02-07T12:55:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi019.ciualv"}]': finished 2024-02-07T12:55:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi172.wueguy"}]: dispatch 2024-02-07T12:55:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi172.wueguy"}]: dispatch 2024-02-07T12:55:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi172.wueguy"}]': finished 2024-02-07T12:55:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.349 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:38.350 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.350 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:38.350 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.350 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:38.350 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.350 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.350 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.350 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.350 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:55:38.351 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:55:38.351 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-02-07T12:55:38.351 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:55:38.351 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:55:38.351 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:55:38.351 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:55:38.351 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-02-07T12:55:38.351 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:55:38.351 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:55:38.351 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-02-07T12:55:38.351 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.351 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.351 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.351 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.351 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.352 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.352 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.352 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.352 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.352 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.352 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.352 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.352 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:55:38.352 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:55:38.352 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-02-07T12:55:38.352 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:55:38.352 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:38.352 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:55:38.352 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:38.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:55:38.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:55:38.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:38.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:55:38.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:55:38.353 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-02-07T12:55:38.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-02-07T12:55:38.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:55:38.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:55:38.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-02-07T12:55:38.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:55:38.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:55:38.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-02-07T12:55:38.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:55:38.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:55:38.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-02-07T12:55:38.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:55:38.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:55:38.354 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-02-07T12:55:38.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:55:38.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:55:38.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-02-07T12:55:38.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:55:38.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:55:38.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-02-07T12:55:38.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:55:38.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:55:38.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:55:38.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:55:38.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-02-07T12:55:38.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:55:38.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:55:38.355 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-02-07T12:55:38.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:55:38.356 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:55:38.357 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-02-07T12:55:38.357 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:55:38.357 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:38.357 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:55:38.357 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:38.357 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:55:38.357 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:55:38.357 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:37 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:38.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:55:38.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:55:38.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi019.ciualv"}]: dispatch 2024-02-07T12:55:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi019.ciualv"}]: dispatch 2024-02-07T12:55:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi019.ciualv"}]': finished 2024-02-07T12:55:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi172.wueguy"}]: dispatch 2024-02-07T12:55:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi172.wueguy"}]: dispatch 2024-02-07T12:55:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi172.wueguy"}]': finished 2024-02-07T12:55:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:55:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:55:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-02-07T12:55:38.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-02-07T12:55:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:55:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:55:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-02-07T12:55:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:55:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:55:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-02-07T12:55:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:55:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:55:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-02-07T12:55:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:55:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:55:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-02-07T12:55:38.372 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:55:38.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:55:38.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-02-07T12:55:38.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:55:38.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:55:38.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-02-07T12:55:38.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:55:38.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:55:38.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:55:38.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:55:38.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-02-07T12:55:38.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:55:38.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:55:38.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-02-07T12:55:38.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:55:38.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:55:38.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:55:38.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-02-07T12:55:38.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:55:38.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:38.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:55:38.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:38.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:55:38.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:55:38.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:37 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:39.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:38 smithi019 ceph-mon[88137]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:39.094 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:55:39.094 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:39.095 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:39.095 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:39.095 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:39.095 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:39.095 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:55:39.095 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:39.095 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:55:39.095 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:39.095 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:55:39.095 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:39.095 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:39.095 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:39.095 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:39.095 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:39.095 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:39.095 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:39.096 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:39.096 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:39.096 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:55:39.096 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:55:39.096 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:55:39.096 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:55:39.096 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:55:39.096 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:55:39.096 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:55:39.096 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:55:39.096 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:55:39.096 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:55:39.096 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:55:39.096 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:55:39.097 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:55:39.097 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:55:39.097 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:55:39.097 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:55:39.097 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:55:39.097 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:39.097 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:55:39.097 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:39.097 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:39.097 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:39.097 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:39.097 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:39.097 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:55:39.097 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:39.097 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:55:39.098 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:39.098 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:55:39.098 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:39.098 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:39.098 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:39.098 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:39.098 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:38 smithi019 ceph-mon[82918]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:39.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:55:39.098 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:39.099 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:39.099 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:39.099 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:39.099 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:39.099 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:55:39.099 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:39.099 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:55:39.099 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:39.099 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:55:39.099 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:39.099 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:39.099 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:39.099 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:39.099 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:39.100 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:39.100 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:39.100 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:39.100 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:39.100 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:55:39.100 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:55:39.100 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:55:39.100 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:55:39.100 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:55:39.100 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:55:39.100 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:55:39.100 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:55:39.100 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:55:39.100 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:55:39.100 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:55:39.101 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:55:39.101 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:55:39.101 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:55:39.101 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:55:39.101 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:55:39.101 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:55:39.101 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:39.101 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:55:39.101 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:39.101 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:39.101 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:39.101 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:39.101 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:39.101 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:55:39.101 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:39.101 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:55:39.102 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:39.102 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:55:39.102 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:39.102 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:39.102 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:39.102 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:39.102 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:39.102 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:39.102 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:39.102 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:39.102 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:39.102 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:38 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all rgw 2024-02-07T12:55:39.102 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:38 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all rbd-mirror 2024-02-07T12:55:39.102 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:38 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all ceph-exporter 2024-02-07T12:55:39.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:38 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all nfs 2024-02-07T12:55:39.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:38 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all nvmeof 2024-02-07T12:55:39.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:38 smithi019 ceph-mon[82918]: Upgrade: Finalizing container_image settings 2024-02-07T12:55:39.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:38 smithi019 ceph-mon[82918]: Upgrade: Complete! 2024-02-07T12:55:39.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:38 smithi019 ceph-mon[82918]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s rd, 0 B/s wr, 73 op/s 2024-02-07T12:55:39.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:38 smithi019 ceph-mon[82918]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 46 op/s 2024-02-07T12:55:39.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:38 smithi019 ceph-mon[82918]: Checking dashboard <-> RGW credentials 2024-02-07T12:55:39.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:38 smithi019 ceph-mon[82918]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:55:39.103 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:38 smithi019 ceph-mon[82918]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:55:39.103 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:39.103 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:39.103 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:39.103 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:39.103 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:39.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:38 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all rgw 2024-02-07T12:55:39.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:38 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all rbd-mirror 2024-02-07T12:55:39.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:38 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all ceph-exporter 2024-02-07T12:55:39.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:38 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all nfs 2024-02-07T12:55:39.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:38 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all nvmeof 2024-02-07T12:55:39.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:38 smithi019 ceph-mon[88137]: Upgrade: Finalizing container_image settings 2024-02-07T12:55:39.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:38 smithi019 ceph-mon[88137]: Upgrade: Complete! 2024-02-07T12:55:39.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:38 smithi019 ceph-mon[88137]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s rd, 0 B/s wr, 73 op/s 2024-02-07T12:55:39.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:38 smithi019 ceph-mon[88137]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 46 op/s 2024-02-07T12:55:39.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:38 smithi019 ceph-mon[88137]: Checking dashboard <-> RGW credentials 2024-02-07T12:55:39.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:38 smithi019 ceph-mon[88137]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:55:39.104 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:38 smithi019 ceph-mon[88137]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:55:39.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:38 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:39.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:55:39.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:39.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:39.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:39.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:39.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:39.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:55:39.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:39.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:55:39.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:39.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:55:39.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:39.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:39.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:39.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:39.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:39.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:39.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:39.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:39.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:39.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:55:39.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:55:39.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:55:39.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:55:39.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:55:39.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:55:39.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:55:39.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:55:39.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:55:39.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:55:39.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:55:39.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:55:39.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:55:39.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:55:39.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:55:39.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:55:39.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:55:39.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:39.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:55:39.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:39.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:39.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:39.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:39.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:39.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:55:39.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:39.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:55:39.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:39.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:55:39.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:39.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:39.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:39.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:39.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:39.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:39.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:39.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:39.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:39.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:38 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all rgw 2024-02-07T12:55:39.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:38 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all rbd-mirror 2024-02-07T12:55:39.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:38 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all ceph-exporter 2024-02-07T12:55:39.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:38 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all nfs 2024-02-07T12:55:39.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:38 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all nvmeof 2024-02-07T12:55:39.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:38 smithi172 ceph-mon[67636]: Upgrade: Finalizing container_image settings 2024-02-07T12:55:39.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:38 smithi172 ceph-mon[67636]: Upgrade: Complete! 2024-02-07T12:55:39.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:38 smithi172 ceph-mon[67636]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s rd, 0 B/s wr, 73 op/s 2024-02-07T12:55:39.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:38 smithi172 ceph-mon[67636]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 46 op/s 2024-02-07T12:55:39.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:38 smithi172 ceph-mon[67636]: Checking dashboard <-> RGW credentials 2024-02-07T12:55:39.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:38 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:55:39.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:38 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:55:39.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:55:39.613+0000 7fb71d671700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:39.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:55:39.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:39.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:39.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:39.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:39.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:39.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:55:39.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:39.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:55:39.845 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:39.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:55:39.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:39.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:39.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:39.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:39.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:39.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:39.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:39.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:39.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:39.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:55:39.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:55:39.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:55:39.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:55:39.846 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:55:39.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:55:39.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:55:39.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:55:39.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:55:39.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:55:39.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:55:39.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:55:39.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:55:39.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:55:39.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:55:39.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:55:39.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:55:39.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:39.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:55:39.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:39.847 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:39.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:39.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:39.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:39.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:55:39.848 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:39.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:55:39.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:39.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:55:39.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:39.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:39.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:39.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:39.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:39.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:39.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:39.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:39.849 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:40.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:55:40.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:40.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:55:40.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:40.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:55:40.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:55:40.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:40.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:55:40.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:40.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:55:40.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:40.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:55:40.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:55:40.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:39 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:40.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:39 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:55:40.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:39 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:40.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:39 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:55:40.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:39 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:40.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:39 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:55:40.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:39 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:55:40.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:39 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:55:41.325 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:40 smithi019 ceph-mon[88137]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-02-07T12:55:41.325 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:40 smithi019 ceph-mon[88137]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:41.325 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:55:41.325 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:41.325 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:41.325 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:41.325 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:41.325 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:41.325 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:55:41.326 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:41.326 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:55:41.326 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:41.326 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:55:41.326 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:41.326 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:41.326 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:41.326 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:41.326 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:41.326 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:41.326 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:41.326 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:41.326 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:41.326 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:55:41.326 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:55:41.326 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:55:41.327 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:55:41.327 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:55:41.327 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:55:41.327 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:55:41.327 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:55:41.327 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:55:41.327 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:55:41.327 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:55:41.327 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:55:41.327 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:55:41.327 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:55:41.327 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:55:41.327 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:55:41.327 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:55:41.327 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:41.327 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:55:41.327 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:41.327 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:41.328 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:41.328 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:41.328 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:41.328 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:55:41.328 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:41.328 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:55:41.328 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:41.328 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:55:41.328 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:41.328 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:41.328 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:41.328 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:41.329 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:40 smithi019 ceph-mon[82918]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-02-07T12:55:41.329 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:40 smithi019 ceph-mon[82918]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:41.329 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:55:41.329 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:41.329 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:41.329 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:41.329 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:41.329 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:41.329 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:55:41.329 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:41.330 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:55:41.330 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:41.330 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:55:41.330 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:41.330 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:41.330 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:41.330 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:41.330 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:41.330 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:41.330 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:41.330 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:41.330 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:41.330 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:55:41.330 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:55:41.331 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:55:41.331 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:55:41.331 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:55:41.331 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:55:41.331 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:55:41.331 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:55:41.331 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:55:41.331 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:55:41.331 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:55:41.331 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:55:41.331 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:55:41.331 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:55:41.331 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:55:41.331 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:55:41.331 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:55:41.332 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:41.332 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:55:41.332 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:41.332 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:41.332 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:41.332 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:41.332 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:41.332 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:55:41.332 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:41.332 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:55:41.332 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:41.332 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:55:41.332 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:41.332 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:41.333 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:41.333 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:41.333 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:41.333 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:41.333 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:41.333 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:41.333 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:41.333 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:40 smithi019 ceph-mon[82918]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 2.8 KiB/s rd, 2 op/s 2024-02-07T12:55:41.333 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:40 smithi019 ceph-mon[82918]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 3 op/s 2024-02-07T12:55:41.333 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:40 smithi019 ceph-mon[82918]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:55:41.333 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:40 smithi019 ceph-mon[82918]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:55:41.333 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:40 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:41.333 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:41.334 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:41.334 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:41.334 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:41.334 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:41.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:40 smithi019 ceph-mon[88137]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 2.8 KiB/s rd, 2 op/s 2024-02-07T12:55:41.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:40 smithi019 ceph-mon[88137]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 3 op/s 2024-02-07T12:55:41.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:40 smithi019 ceph-mon[88137]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:55:41.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:40 smithi019 ceph-mon[88137]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:55:41.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:40 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:41.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:40 smithi172 ceph-mon[67636]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-02-07T12:55:41.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:40 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:41.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:55:41.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:41.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:41.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:41.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:41.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:41.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:55:41.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:41.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:55:41.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:41.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:55:41.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:41.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:41.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:41.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:41.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:41.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:41.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:41.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:41.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:41.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:55:41.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:55:41.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:55:41.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:55:41.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:55:41.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:55:41.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:55:41.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:55:41.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:55:41.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:55:41.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:55:41.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:55:41.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:55:41.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:55:41.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:55:41.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:55:41.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:55:41.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:41.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:55:41.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:41.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:41.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:41.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:41.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:41.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:55:41.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:41.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:55:41.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:41.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:55:41.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:41.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:41.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:41.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:41.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:41.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:41.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:41.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:41.374 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:41.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:40 smithi172 ceph-mon[67636]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 2.8 KiB/s rd, 2 op/s 2024-02-07T12:55:41.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:40 smithi172 ceph-mon[67636]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 3 op/s 2024-02-07T12:55:41.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:40 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:55:41.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:40 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:55:41.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:40 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:55:41.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:55:41.441+0000 7fb71d671700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:41.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:55:41.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:41.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:41.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:41.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:41.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:41.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:55:41.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:41.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:55:41.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:41.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:55:41.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:41.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:41.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:41.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:41.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:41.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:41.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:41.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:41.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:41.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:55:41.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:55:41.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:55:41.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:55:41.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:55:41.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:55:41.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:55:41.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:55:41.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:55:41.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:55:41.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:55:41.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:55:41.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:55:41.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:55:41.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:55:41.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:55:41.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:55:41.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:41.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:55:41.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:41.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:41.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:41.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:41.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:41.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:55:41.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:41.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:55:41.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:41.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:55:41.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:41.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:41.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:41.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:41.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:41.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:41.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:41.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:41.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:41 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:43.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:42 smithi019 ceph-mon[88137]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:43.344 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:55:43.344 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:43.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:43.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:43.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:43.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:43.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:55:43.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:43.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:55:43.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:43.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:55:43.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:43.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:43.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:43.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:43.345 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:43.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:43.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:43.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:43.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:43.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:55:43.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:55:43.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:55:43.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:55:43.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:55:43.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:55:43.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:55:43.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:55:43.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:55:43.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:55:43.346 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:55:43.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:55:43.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:55:43.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:55:43.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:55:43.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:55:43.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:55:43.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:43.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:55:43.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:43.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:43.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:43.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:43.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:43.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:55:43.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:43.347 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:55:43.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:43.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:55:43.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:43.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:43.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:43.348 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:43.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:42 smithi019 ceph-mon[82918]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:43.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:55:43.348 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:43.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:43.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:43.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:43.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:43.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:55:43.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:43.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:55:43.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:43.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:55:43.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:43.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:43.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:43.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:43.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:43.349 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:43.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:43.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:43.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:43.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:55:43.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:55:43.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:55:43.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:55:43.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:55:43.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:55:43.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:55:43.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:55:43.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:55:43.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:55:43.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:55:43.350 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:55:43.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:55:43.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:55:43.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:55:43.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:55:43.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:55:43.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:43.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:55:43.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:43.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:43.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:43.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:43.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:43.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:55:43.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:43.351 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:55:43.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:43.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:55:43.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:43.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:43.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:43.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:43.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:43.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:43.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:43.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:43.352 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:43.352 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:42 smithi019 ceph-mon[82918]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 2 op/s 2024-02-07T12:55:43.352 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:43.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:43.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:43.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:43.353 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:43.353 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:42 smithi019 ceph-mon[88137]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 2 op/s 2024-02-07T12:55:43.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:42 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:43.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:55:43.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:43.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:43.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:43.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:43.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:43.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:55:43.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:43.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:55:43.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:43.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:55:43.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:43.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:43.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:43.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:43.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:43.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:43.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:43.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:43.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:43.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:55:43.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:55:43.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:55:43.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:55:43.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:55:43.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:55:43.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:55:43.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:55:43.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:55:43.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:55:43.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:55:43.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:55:43.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:55:43.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:55:43.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:55:43.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:55:43.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:55:43.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:55:43.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:55:43.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:55:43.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:55:43.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:55:43.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:43.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:43.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:55:43.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:55:43.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:55:43.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:55:43.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:55:43.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:55:43.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:55:43.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:55:43.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:55:43.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:55:43.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:55:43.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:55:43.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:55:43.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:55:43.374 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:42 smithi172 ceph-mon[67636]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 2 op/s 2024-02-07T12:55:44.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:55:43] "GET /metrics HTTP/1.1" 200 38406 "" "Prometheus/2.43.0" 2024-02-07T12:55:45.162 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:44 smithi019 ceph-mon[88137]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 5.2 KiB/s rd, 5 op/s 2024-02-07T12:55:45.162 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:44 smithi019 ceph-mon[82918]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 5.2 KiB/s rd, 5 op/s 2024-02-07T12:55:45.185 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch ps' 2024-02-07T12:55:45.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:44 smithi172 ceph-mon[67636]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 5.2 KiB/s rd, 5 op/s 2024-02-07T12:55:46.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:45 smithi019 ceph-mon[82918]: from='client.44760 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:55:46.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:45 smithi019 ceph-mon[88137]: from='client.44760 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:55:46.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:45 smithi172 ceph-mon[67636]: from='client.44760 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:55:47.127 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:55:47.127 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (10m) 16s ago 31m 26.9M - 0.25.0 c8568f914cd2 fec9255f35e7 2024-02-07T12:55:47.127 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (9m) 16s ago 31m 86.0M - 9.4.12 a0ea5e3c46d9 aaffac63bc28 2024-02-07T12:55:47.127 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 unknown 16s ago 31m 63.7M - 3.5 e1d6a67b021e 5113aa77e3f2 2024-02-07T12:55:47.128 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283,8765 running (11m) 16s ago 36m 452M - 19.0.0-1252-g44458cce 79957b372307 5aa0c04fdfd7 2024-02-07T12:55:47.128 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (21m) 16s ago 38m 515M - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:55:47.128 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (7m) 16s ago 38m 62.5M 2048M 19.0.0-1252-g44458cce 79957b372307 8b1b0e21b1c5 2024-02-07T12:55:47.128 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (8m) 16s ago 36m 56.4M 2048M 19.0.0-1252-g44458cce 79957b372307 00f454bbac01 2024-02-07T12:55:47.128 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (7m) 16s ago 36m 50.3M 2048M 19.0.0-1252-g44458cce 79957b372307 6214003f6e08 2024-02-07T12:55:47.128 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (10m) 16s ago 32m 15.1M - 1.5.0 0da6a335fe13 c4bdcb1c4d49 2024-02-07T12:55:47.128 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (10m) 16s ago 32m 16.4M - 1.5.0 0da6a335fe13 04ea81f79d85 2024-02-07T12:55:47.128 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (5m) 16s ago 36m 63.9M 2993M 19.0.0-1252-g44458cce 79957b372307 78eddd34a903 2024-02-07T12:55:47.128 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (5m) 16s ago 35m 54.3M 2993M 19.0.0-1252-g44458cce 79957b372307 c37447e76c38 2024-02-07T12:55:47.128 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (4m) 16s ago 35m 55.7M 2993M 19.0.0-1252-g44458cce 79957b372307 90352897da85 2024-02-07T12:55:47.128 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (3m) 16s ago 34m 62.2M 2993M 19.0.0-1252-g44458cce 79957b372307 067cd6b3fd2f 2024-02-07T12:55:47.128 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (3m) 16s ago 34m 51.5M 3248M 19.0.0-1252-g44458cce 79957b372307 dbb47a8ad5ac 2024-02-07T12:55:47.128 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (2m) 16s ago 34m 48.4M 3248M 19.0.0-1252-g44458cce 79957b372307 d139d35fec57 2024-02-07T12:55:47.128 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (108s) 16s ago 33m 54.0M 3248M 19.0.0-1252-g44458cce 79957b372307 219cf25240d5 2024-02-07T12:55:47.129 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (71s) 16s ago 33m 50.0M 3248M 19.0.0-1252-g44458cce 79957b372307 75dab2b6606b 2024-02-07T12:55:47.129 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (10m) 16s ago 32m 71.7M - 2.43.0 a07b618ecd1d c9b607eea1be 2024-02-07T12:55:47.129 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (26s) 16s ago 31m 76.4M - 19.0.0-1252-g44458cce 79957b372307 3753defe9ced 2024-02-07T12:55:47.129 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (20s) 16s ago 31m 76.4M - 19.0.0-1252-g44458cce 79957b372307 6e879570d813 2024-02-07T12:55:47.129 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:46 smithi019 ceph-mon[82918]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 4 op/s 2024-02-07T12:55:47.129 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:46 smithi019 ceph-mon[88137]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 4 op/s 2024-02-07T12:55:47.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:46 smithi172 ceph-mon[67636]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 4 op/s 2024-02-07T12:55:47.768 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph versions | jq -e '"'"'.rgw | length == 1'"'"'' 2024-02-07T12:55:48.082 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:47 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:55:48.082 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:47 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:55:48.270 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:47 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:55:49.253 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:48 smithi019 ceph-mon[82918]: from='client.44766 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:55:49.253 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:48 smithi019 ceph-mon[82918]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-02-07T12:55:49.253 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:48 smithi019 ceph-mon[88137]: from='client.44766 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:55:49.253 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:48 smithi019 ceph-mon[88137]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-02-07T12:55:49.286 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:48 smithi172 ceph-mon[67636]: from='client.44766 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:55:49.286 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:48 smithi172 ceph-mon[67636]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-02-07T12:55:49.816 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:55:50.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:49 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/4120762011' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:50.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:49 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/4120762011' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:50.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:49 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/4120762011' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:50.421 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph versions | jq -e '"'"'.rgw | keys'"'"' | grep $sha1' 2024-02-07T12:55:51.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:50 smithi019 ceph-mon[88137]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 2 op/s 2024-02-07T12:55:51.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:50 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:55:51.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:50 smithi019 ceph-mon[82918]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 2 op/s 2024-02-07T12:55:51.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:50 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:55:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:50 smithi172 ceph-mon[67636]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 2 op/s 2024-02-07T12:55:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:50 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:55:52.537 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)" 2024-02-07T12:55:53.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:52 smithi019 ceph-mon[88137]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 2 op/s 2024-02-07T12:55:53.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:52 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/1579134528' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:53.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:52 smithi019 ceph-mon[82918]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 2 op/s 2024-02-07T12:55:53.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:52 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/1579134528' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:53.169 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch upgrade status' 2024-02-07T12:55:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:52 smithi172 ceph-mon[67636]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 2 op/s 2024-02-07T12:55:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:52 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/1579134528' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:55:54.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:55:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:55:53] "GET /metrics HTTP/1.1" 200 38406 "" "Prometheus/2.43.0" 2024-02-07T12:55:55.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:54 smithi019 ceph-mon[88137]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 2 op/s 2024-02-07T12:55:55.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:54 smithi019 ceph-mon[82918]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 2 op/s 2024-02-07T12:55:55.264 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:55:55.264 INFO:teuthology.orchestra.run.smithi019.stdout: "target_image": null, 2024-02-07T12:55:55.264 INFO:teuthology.orchestra.run.smithi019.stdout: "in_progress": false, 2024-02-07T12:55:55.265 INFO:teuthology.orchestra.run.smithi019.stdout: "which": "", 2024-02-07T12:55:55.265 INFO:teuthology.orchestra.run.smithi019.stdout: "services_complete": [], 2024-02-07T12:55:55.265 INFO:teuthology.orchestra.run.smithi019.stdout: "progress": null, 2024-02-07T12:55:55.265 INFO:teuthology.orchestra.run.smithi019.stdout: "message": "", 2024-02-07T12:55:55.265 INFO:teuthology.orchestra.run.smithi019.stdout: "is_paused": false 2024-02-07T12:55:55.265 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:55:55.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:54 smithi172 ceph-mon[67636]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 2 op/s 2024-02-07T12:55:55.979 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph health detail' 2024-02-07T12:55:57.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:56 smithi019 ceph-mon[88137]: from='client.54724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:55:57.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:56 smithi019 ceph-mon[88137]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:55:57.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:56 smithi019 ceph-mon[82918]: from='client.54724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:55:57.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:56 smithi019 ceph-mon[82918]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:55:57.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:56 smithi172 ceph-mon[67636]: from='client.54724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:55:57.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:56 smithi172 ceph-mon[67636]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:55:58.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:57 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:55:58.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:57 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:55:58.263 INFO:teuthology.orchestra.run.smithi019.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-02-07T12:55:58.263 INFO:teuthology.orchestra.run.smithi019.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-02-07T12:55:58.263 INFO:teuthology.orchestra.run.smithi019.stdout: daemon iscsi.foo.smithi019.svjimj on smithi019 is in unknown state 2024-02-07T12:55:58.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:57 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:55:58.890 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-02-07T12:55:59.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:59 smithi019 ceph-mon[82918]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:55:59.334 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:55:59 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/3662601101' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:55:59.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:59 smithi019 ceph-mon[88137]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:55:59.334 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:55:59 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/3662601101' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:55:59.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:59 smithi172 ceph-mon[67636]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:55:59.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:55:59 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/3662601101' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:56:00.898 INFO:teuthology.orchestra.run.smithi019.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:56:01.264 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:01 smithi019 ceph-mon[88137]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:56:01.264 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:01 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:01.264 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:01 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:56:01.264 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:01 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:56:01.264 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:01 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:56:01.264 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:01 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:01.265 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:01 smithi019 ceph-mon[82918]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:56:01.265 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:01 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:01.265 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:01 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:56:01.265 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:01 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:56:01.265 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:01 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:56:01.265 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:01 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:01.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:01 smithi172 ceph-mon[67636]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:56:01.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:01 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:01.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:01 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:56:01.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:01 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:56:01.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:01 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:56:01.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:01 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:01.580 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-02-07T12:56:01.591 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi019.front.sepia.ceph.com 2024-02-07T12:56:01.591 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-02-07T12:56:01.869 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:56:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a[66030]: logger=cleanup t=2024-02-07T12:56:01.507205267Z level=info msg="Completed cleanup jobs" duration=1.42982ms 2024-02-07T12:56:02.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:02 smithi019 ceph-mon[88137]: from='client.44796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:56:02.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:02 smithi019 ceph-mon[88137]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:56:02.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:02 smithi019 ceph-mon[88137]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 907 B/s rd, 0 op/s 2024-02-07T12:56:02.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:02 smithi019 ceph-mon[88137]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:56:02.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:02 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:56:02.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:02 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:56:02.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:02 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:56:02.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:02 smithi019 ceph-mon[82918]: from='client.44796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:56:02.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:02 smithi019 ceph-mon[82918]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:56:02.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:02 smithi019 ceph-mon[82918]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 907 B/s rd, 0 op/s 2024-02-07T12:56:02.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:02 smithi019 ceph-mon[82918]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:56:02.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:02 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:56:02.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:02 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:56:02.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:02 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:56:02.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:02 smithi172 ceph-mon[67636]: from='client.44796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:56:02.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:02 smithi172 ceph-mon[67636]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:56:02.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:02 smithi172 ceph-mon[67636]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 907 B/s rd, 0 op/s 2024-02-07T12:56:02.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:02 smithi172 ceph-mon[67636]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:56:02.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:02 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:56:02.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:02 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:56:02.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:02 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:56:03.317 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-mon[88137]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:56:03.318 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-mon[88137]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:56:03.318 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-mon[82918]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:56:03.318 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-mon[82918]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:56:03.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:03 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:56:03.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:03 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:56:03.594 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:56:03.464+0000 7fb71d671700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:56:03.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:56:03.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:56:03.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:56:03.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:56:03.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:56:03.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:56:03.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:56:03.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:56:03.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:56:03.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:56:03.595 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:56:03.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:56:03.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:56:03.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:56:03.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:56:03.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:56:03.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:56:03.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:56:03.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:56:03.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:56:03.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:56:03.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:56:03.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:56:03.596 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:56:03.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:56:03.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:56:03.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:56:03.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:56:03.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return future.result(timeout) 2024-02-07T12:56:03.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:56:03.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return self.__get_result() 2024-02-07T12:56:03.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:56:03.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: raise self._exception 2024-02-07T12:56:03.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:56:03.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: config_blobs=daemon_spec.final_config, 2024-02-07T12:56:03.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:56:03.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:56:03.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:56:03.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Creating ceph-iscsi config... 2024-02-07T12:56:03.597 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:56:03.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:56:03.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:56:03.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:56:03.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:56:03.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: Traceback (most recent call last): 2024-02-07T12:56:03.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:56:03.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: return _run_code(code, main_globals, None, 2024-02-07T12:56:03.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:56:03.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: exec(code, run_globals) 2024-02-07T12:56:03.598 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:56:03.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:56:03.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:56:03.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:56:03.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:56:03.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:56:03.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:56:03.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:56:03.599 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:56:04.126 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:56:03] "GET /metrics HTTP/1.1" 200 38406 "" "Prometheus/2.43.0" 2024-02-07T12:56:04.126 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:04 smithi019 ceph-mon[88137]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 551 B/s rd, 0 op/s 2024-02-07T12:56:04.127 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:04 smithi019 ceph-mon[88137]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:56:04.127 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:56:04.127 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:56:04.127 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:56:04.127 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:56:04.127 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:56:04.127 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:56:04.127 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:56:04.127 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:56:04.127 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:56:04.127 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:56:04.128 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:56:04.128 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:56:04.128 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:56:04.128 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:56:04.128 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:56:04.128 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:56:04.128 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:56:04.128 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:56:04.128 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:56:04.128 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:56:04.128 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:56:04.128 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:56:04.129 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:04 smithi019 ceph-mon[82918]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 551 B/s rd, 0 op/s 2024-02-07T12:56:04.129 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:04 smithi019 ceph-mon[82918]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:56:04.129 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:56:04.129 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:56:04.129 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:56:04.129 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:56:04.129 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:56:04.129 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:56:04.129 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:56:04.129 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:56:04.130 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:56:04.130 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:56:04.130 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:56:04.130 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:56:04.130 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:56:04.130 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:56:04.130 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:56:04.130 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:56:04.130 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:56:04.130 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:56:04.130 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:56:04.130 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:56:04.130 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:56:04.130 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:56:04.130 INFO:journalctl@ceph.mon.a.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:56:04.131 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:56:04.131 INFO:journalctl@ceph.mon.a.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:56:04.131 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:56:04.131 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:56:04.131 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:56:04.131 INFO:journalctl@ceph.mon.a.smithi019.stdout: return future.result(timeout) 2024-02-07T12:56:04.131 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:56:04.131 INFO:journalctl@ceph.mon.a.smithi019.stdout: return self.__get_result() 2024-02-07T12:56:04.131 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:56:04.131 INFO:journalctl@ceph.mon.a.smithi019.stdout: raise self._exception 2024-02-07T12:56:04.131 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:56:04.131 INFO:journalctl@ceph.mon.a.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:56:04.131 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:56:04.131 INFO:journalctl@ceph.mon.a.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:56:04.132 INFO:journalctl@ceph.mon.a.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:56:04.132 INFO:journalctl@ceph.mon.a.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:56:04.132 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:56:04.132 INFO:journalctl@ceph.mon.a.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:56:04.132 INFO:journalctl@ceph.mon.a.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:56:04.132 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:56:04.132 INFO:journalctl@ceph.mon.a.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:56:04.132 INFO:journalctl@ceph.mon.a.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:56:04.132 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:56:04.132 INFO:journalctl@ceph.mon.a.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:56:04.132 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:56:04.132 INFO:journalctl@ceph.mon.a.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:56:04.132 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:56:04.133 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:56:04.133 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:56:04.133 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:56:04.133 INFO:journalctl@ceph.mon.c.smithi019.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:56:04.133 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:56:04.133 INFO:journalctl@ceph.mon.c.smithi019.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:56:04.133 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:56:04.133 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:56:04.133 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:56:04.133 INFO:journalctl@ceph.mon.c.smithi019.stdout: return future.result(timeout) 2024-02-07T12:56:04.134 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:56:04.134 INFO:journalctl@ceph.mon.c.smithi019.stdout: return self.__get_result() 2024-02-07T12:56:04.134 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:56:04.134 INFO:journalctl@ceph.mon.c.smithi019.stdout: raise self._exception 2024-02-07T12:56:04.134 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:56:04.134 INFO:journalctl@ceph.mon.c.smithi019.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:56:04.134 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:56:04.134 INFO:journalctl@ceph.mon.c.smithi019.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:56:04.134 INFO:journalctl@ceph.mon.c.smithi019.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:56:04.134 INFO:journalctl@ceph.mon.c.smithi019.stdout: Creating ceph-iscsi config... 2024-02-07T12:56:04.134 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:56:04.134 INFO:journalctl@ceph.mon.c.smithi019.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:56:04.134 INFO:journalctl@ceph.mon.c.smithi019.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:56:04.134 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:56:04.135 INFO:journalctl@ceph.mon.c.smithi019.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:56:04.135 INFO:journalctl@ceph.mon.c.smithi019.stdout: Traceback (most recent call last): 2024-02-07T12:56:04.135 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:56:04.135 INFO:journalctl@ceph.mon.c.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:56:04.135 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:56:04.135 INFO:journalctl@ceph.mon.c.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:56:04.135 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:56:04.135 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:56:04.135 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:56:04.135 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:56:04.135 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:56:04.135 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:56:04.135 INFO:journalctl@ceph.mon.c.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:56:04.135 INFO:journalctl@ceph.mon.c.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:56:04.135 INFO:journalctl@ceph.mon.c.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:56:04.136 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:04 smithi019 ceph-mon[88137]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:56:04.136 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:56:04.136 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:56:04.136 INFO:journalctl@ceph.mon.a.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:56:04.136 INFO:journalctl@ceph.mon.a.smithi019.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:56:04.136 INFO:journalctl@ceph.mon.a.smithi019.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:56:04.136 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:04 smithi019 ceph-mon[82918]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:56:04.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:04 smithi172 ceph-mon[67636]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 551 B/s rd, 0 op/s 2024-02-07T12:56:04.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:04 smithi172 ceph-mon[67636]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:56:04.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:56:04.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:56:04.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:56:04.369 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:56:04.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:56:04.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:56:04.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:56:04.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:56:04.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:56:04.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:56:04.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:56:04.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:56:04.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:56:04.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:56:04.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:56:04.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:56:04.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:56:04.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:56:04.370 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:56:04.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:56:04.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:56:04.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1102, in _check_daemons 2024-02-07T12:56:04.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-02-07T12:56:04.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2368, in _daemon_action 2024-02-07T12:56:04.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-02-07T12:56:04.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 739, in wait_async 2024-02-07T12:56:04.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-02-07T12:56:04.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result 2024-02-07T12:56:04.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-02-07T12:56:04.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-02-07T12:56:04.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-02-07T12:56:04.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-02-07T12:56:04.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-02-07T12:56:04.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1362, in _create_daemon 2024-02-07T12:56:04.371 INFO:journalctl@ceph.mon.b.smithi172.stdout: config_blobs=daemon_spec.final_config, 2024-02-07T12:56:04.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1664, in _run_cephadm 2024-02-07T12:56:04.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-02-07T12:56:04.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:56:04.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-02-07T12:56:04.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:56:04.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:56:04.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:56:04.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:56:04.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:56:04.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-02-07T12:56:04.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:56:04.372 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:56:04.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:56:04.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-02-07T12:56:04.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:56:04.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:56:04.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:56:04.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:56:04.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:56:04.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1065, in deploy_daemon 2024-02-07T12:56:04.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:56:04.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:56:04.373 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:56:04.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:04 smithi172 ceph-mon[67636]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:56:04.599 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:56:05.327 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:56:05.328 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (10m) 34s ago 32m 26.9M - 0.25.0 c8568f914cd2 fec9255f35e7 2024-02-07T12:56:05.328 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (10m) 34s ago 31m 86.0M - 9.4.12 a0ea5e3c46d9 aaffac63bc28 2024-02-07T12:56:05.328 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 unknown 34s ago 31m 63.7M - 3.5 e1d6a67b021e 5113aa77e3f2 2024-02-07T12:56:05.328 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283,8765 running (11m) 34s ago 36m 452M - 19.0.0-1252-g44458cce 79957b372307 5aa0c04fdfd7 2024-02-07T12:56:05.328 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (21m) 34s ago 38m 515M - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:56:05.328 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (8m) 34s ago 38m 62.5M 2048M 19.0.0-1252-g44458cce 79957b372307 8b1b0e21b1c5 2024-02-07T12:56:05.328 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (9m) 34s ago 36m 56.4M 2048M 19.0.0-1252-g44458cce 79957b372307 00f454bbac01 2024-02-07T12:56:05.328 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (7m) 34s ago 37m 50.3M 2048M 19.0.0-1252-g44458cce 79957b372307 6214003f6e08 2024-02-07T12:56:05.328 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (11m) 34s ago 32m 15.1M - 1.5.0 0da6a335fe13 c4bdcb1c4d49 2024-02-07T12:56:05.328 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (10m) 34s ago 32m 16.4M - 1.5.0 0da6a335fe13 04ea81f79d85 2024-02-07T12:56:05.328 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (6m) 34s ago 36m 63.9M 2993M 19.0.0-1252-g44458cce 79957b372307 78eddd34a903 2024-02-07T12:56:05.328 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (5m) 34s ago 35m 54.3M 2993M 19.0.0-1252-g44458cce 79957b372307 c37447e76c38 2024-02-07T12:56:05.328 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (4m) 34s ago 35m 55.7M 2993M 19.0.0-1252-g44458cce 79957b372307 90352897da85 2024-02-07T12:56:05.328 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (3m) 34s ago 35m 62.2M 2993M 19.0.0-1252-g44458cce 79957b372307 067cd6b3fd2f 2024-02-07T12:56:05.328 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (3m) 34s ago 34m 51.5M 3248M 19.0.0-1252-g44458cce 79957b372307 dbb47a8ad5ac 2024-02-07T12:56:05.329 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (2m) 34s ago 34m 48.4M 3248M 19.0.0-1252-g44458cce 79957b372307 d139d35fec57 2024-02-07T12:56:05.329 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (2m) 34s ago 34m 54.0M 3248M 19.0.0-1252-g44458cce 79957b372307 219cf25240d5 2024-02-07T12:56:05.329 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (89s) 34s ago 33m 50.0M 3248M 19.0.0-1252-g44458cce 79957b372307 75dab2b6606b 2024-02-07T12:56:05.329 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (10m) 34s ago 33m 71.7M - 2.43.0 a07b618ecd1d c9b607eea1be 2024-02-07T12:56:05.329 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (44s) 34s ago 31m 76.4M - 19.0.0-1252-g44458cce 79957b372307 3753defe9ced 2024-02-07T12:56:05.329 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (38s) 34s ago 31m 76.4M - 19.0.0-1252-g44458cce 79957b372307 6e879570d813 2024-02-07T12:56:05.747 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:56:05.747 INFO:teuthology.orchestra.run.smithi019.stdout: "mon": { 2024-02-07T12:56:05.747 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 3 2024-02-07T12:56:05.747 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:56:05.747 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr": { 2024-02-07T12:56:05.747 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:56:05.747 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:56:05.747 INFO:teuthology.orchestra.run.smithi019.stdout: "osd": { 2024-02-07T12:56:05.748 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 8 2024-02-07T12:56:05.748 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:56:05.748 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw": { 2024-02-07T12:56:05.748 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:56:05.748 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:56:05.748 INFO:teuthology.orchestra.run.smithi019.stdout: "overall": { 2024-02-07T12:56:05.748 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 15 2024-02-07T12:56:05.748 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:56:05.748 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:56:06.128 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:56:06.128 INFO:teuthology.orchestra.run.smithi019.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:56:06.128 INFO:teuthology.orchestra.run.smithi019.stdout: "in_progress": true, 2024-02-07T12:56:06.128 INFO:teuthology.orchestra.run.smithi019.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-07T12:56:06.128 INFO:teuthology.orchestra.run.smithi019.stdout: "services_complete": [ 2024-02-07T12:56:06.128 INFO:teuthology.orchestra.run.smithi019.stdout: "osd", 2024-02-07T12:56:06.128 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr", 2024-02-07T12:56:06.128 INFO:teuthology.orchestra.run.smithi019.stdout: "mon", 2024-02-07T12:56:06.128 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw" 2024-02-07T12:56:06.128 INFO:teuthology.orchestra.run.smithi019.stdout: ], 2024-02-07T12:56:06.129 INFO:teuthology.orchestra.run.smithi019.stdout: "progress": "15/21 daemons upgraded", 2024-02-07T12:56:06.129 INFO:teuthology.orchestra.run.smithi019.stdout: "message": "Currently upgrading iscsi daemons", 2024-02-07T12:56:06.129 INFO:teuthology.orchestra.run.smithi019.stdout: "is_paused": false 2024-02-07T12:56:06.129 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:56:06.170 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:05 smithi019 ceph-mon[88137]: from='client.44802 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:56:06.170 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:05 smithi019 ceph-mon[88137]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:56:06.171 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:05 smithi019 ceph-mon[88137]: from='client.44808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:56:06.171 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:56:06.171 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:05 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/457652688' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:06.171 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:05 smithi019 ceph-mon[82918]: from='client.44802 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:56:06.171 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:05 smithi019 ceph-mon[82918]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:56:06.171 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:05 smithi019 ceph-mon[82918]: from='client.44808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:56:06.171 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:56:06.171 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:05 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/457652688' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:05 smithi172 ceph-mon[67636]: from='client.44802 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:56:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:05 smithi172 ceph-mon[67636]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:56:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:05 smithi172 ceph-mon[67636]: from='client.44808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:56:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:56:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:05 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/457652688' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:06.540 INFO:teuthology.orchestra.run.smithi019.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-02-07T12:56:06.540 INFO:teuthology.orchestra.run.smithi019.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-02-07T12:56:06.540 INFO:teuthology.orchestra.run.smithi019.stdout: daemon iscsi.foo.smithi019.svjimj on smithi019 is in unknown state 2024-02-07T12:56:07.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[88137]: from='client.44814 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:56:07.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:07.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[88137]: Upgrade: Target is version 19.0.0-1252-g44458cce (squid) 2024-02-07T12:56:07.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[88137]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe'] 2024-02-07T12:56:07.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:56:07.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:07.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all mgr 2024-02-07T12:56:07.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:07.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:07.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all mon 2024-02-07T12:56:07.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:07.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:07.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:07.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:07.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:07.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:07.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:07.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:07.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:07.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:07.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:07.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:07.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:07.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:07.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/3806749459' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:56:07.347 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[82918]: from='client.44814 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:56:07.347 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:07.347 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[82918]: Upgrade: Target is version 19.0.0-1252-g44458cce (squid) 2024-02-07T12:56:07.347 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[82918]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe'] 2024-02-07T12:56:07.347 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:56:07.347 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:07.347 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all mgr 2024-02-07T12:56:07.347 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:07.347 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:07.347 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all mon 2024-02-07T12:56:07.347 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:07.347 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:07.347 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:07.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:07.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:07.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:07.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:07.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:07.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:07.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:07.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:07.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:07.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:07.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:07.348 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:06 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/3806749459' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:56:07.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:06 smithi172 ceph-mon[67636]: from='client.44814 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:56:07.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:06 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:07.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:06 smithi172 ceph-mon[67636]: Upgrade: Target is version 19.0.0-1252-g44458cce (squid) 2024-02-07T12:56:07.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:06 smithi172 ceph-mon[67636]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe'] 2024-02-07T12:56:07.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:06 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:56:07.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:06 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:06 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all mgr 2024-02-07T12:56:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:06 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:06 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:06 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all mon 2024-02-07T12:56:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:06 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:06 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:06 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:06 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:06 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:06 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:07.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:06 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:07.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:06 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:07.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:06 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:07.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:06 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:07.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:06 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:07.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:06 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:07.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:06 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:07.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:06 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:07.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:06 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/3806749459' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:56:08.303 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all crash 2024-02-07T12:56:08.303 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all osd 2024-02-07T12:56:08.303 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all mds 2024-02-07T12:56:08.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all rgw 2024-02-07T12:56:08.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all rbd-mirror 2024-02-07T12:56:08.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all cephfs-mirror 2024-02-07T12:56:08.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[88137]: Upgrade: Setting container_image for all ceph-exporter 2024-02-07T12:56:08.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[88137]: from='client.44826 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:56:08.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:56:08.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[88137]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:56:08.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:08.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:56:08.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:56:08.304 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:56:08.304 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all crash 2024-02-07T12:56:08.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all osd 2024-02-07T12:56:08.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all mds 2024-02-07T12:56:08.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all rgw 2024-02-07T12:56:08.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all rbd-mirror 2024-02-07T12:56:08.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all cephfs-mirror 2024-02-07T12:56:08.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[82918]: Upgrade: Setting container_image for all ceph-exporter 2024-02-07T12:56:08.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[82918]: from='client.44826 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:56:08.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:56:08.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[82918]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:56:08.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:08.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:56:08.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:56:08.305 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:07 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:56:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:07 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all crash 2024-02-07T12:56:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:07 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all osd 2024-02-07T12:56:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:07 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all mds 2024-02-07T12:56:08.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:07 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all rgw 2024-02-07T12:56:08.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:07 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all rbd-mirror 2024-02-07T12:56:08.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:07 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all cephfs-mirror 2024-02-07T12:56:08.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:07 smithi172 ceph-mon[67636]: Upgrade: Setting container_image for all ceph-exporter 2024-02-07T12:56:08.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:07 smithi172 ceph-mon[67636]: from='client.44826 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:56:08.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:07 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:56:08.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:07 smithi172 ceph-mon[67636]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:56:08.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:07 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:08.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:07 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:56:08.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:07 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:56:08.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:07 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:56:09.332 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:09 smithi019 ceph-mon[82918]: Upgrade: Updating iscsi.foo.smithi019.svjimj 2024-02-07T12:56:09.332 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:09 smithi019 ceph-mon[82918]: Deploying daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:56:09.333 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:09 smithi019 ceph-mon[88137]: Upgrade: Updating iscsi.foo.smithi019.svjimj 2024-02-07T12:56:09.333 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:09 smithi019 ceph-mon[88137]: Deploying daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:56:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:09 smithi172 ceph-mon[67636]: Upgrade: Updating iscsi.foo.smithi019.svjimj 2024-02-07T12:56:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:09 smithi172 ceph-mon[67636]: Deploying daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:56:10.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:10 smithi019 ceph-mon[82918]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:56:10.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:10 smithi019 ceph-mon[88137]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:56:10.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:10 smithi172 ceph-mon[67636]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:56:12.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:12 smithi019 ceph-mon[82918]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:56:12.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:12 smithi019 ceph-mon[88137]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:56:12.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:12 smithi172 ceph-mon[67636]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:56:14.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:14 smithi019 ceph-mon[82918]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:56:14.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:14 smithi019 ceph-mon[88137]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:56:14.345 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:56:13] "GET /metrics HTTP/1.1" 200 38403 "" "Prometheus/2.43.0" 2024-02-07T12:56:14.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:14 smithi172 ceph-mon[67636]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:56:16.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:16 smithi019 ceph-mon[82918]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:56:16.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:16 smithi019 ceph-mon[88137]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:56:16.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:16 smithi172 ceph-mon[67636]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:56:18.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:18 smithi019 ceph-mon[88137]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:56:18.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:18 smithi019 ceph-mon[88137]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:56:18.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:18 smithi019 ceph-mon[82918]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:56:18.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:18 smithi019 ceph-mon[82918]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:56:18.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:18 smithi172 ceph-mon[67636]: from='client.15240 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:56:18.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:18 smithi172 ceph-mon[67636]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:56:20.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:20 smithi019 ceph-mon[82918]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:56:20.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:20 smithi019 ceph-mon[88137]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:56:20.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:20 smithi172 ceph-mon[67636]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:56:21.026 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:20 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:56:20.742+0000 7fb71d671700 -1 log_channel(cephadm) log [ERR] : Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi019.svjimj on host smithi019 failed. 2024-02-07T12:56:21.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:56:21.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[82918]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi019.svjimj on host smithi019 failed. 2024-02-07T12:56:21.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:21.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:56:21.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[82918]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 0 op/s 2024-02-07T12:56:21.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:56:21.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:56:21.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[82918]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:56:21.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[82918]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-02-07T12:56:21.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:21.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[82918]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:56:21.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:56:21.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:56:21.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:56:21.346 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[82918]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:56:21.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:56:21.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[88137]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi019.svjimj on host smithi019 failed. 2024-02-07T12:56:21.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:21.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:56:21.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[88137]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 0 op/s 2024-02-07T12:56:21.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:56:21.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:56:21.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[88137]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:56:21.346 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[88137]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-02-07T12:56:21.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:21.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[88137]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:56:21.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:56:21.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:56:21.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:56:21.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:21 smithi019 ceph-mon[88137]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:56:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:21 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:56:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:21 smithi172 ceph-mon[67636]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi019.svjimj on host smithi019 failed. 2024-02-07T12:56:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:21 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:21 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:56:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:21 smithi172 ceph-mon[67636]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 0 op/s 2024-02-07T12:56:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:21 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:56:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:21 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:56:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:21 smithi172 ceph-mon[67636]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-02-07T12:56:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:21 smithi172 ceph-mon[67636]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-02-07T12:56:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:21 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:21 smithi172 ceph-mon[67636]: Reconfiguring iscsi.foo.smithi019.svjimj (dependencies changed)... 2024-02-07T12:56:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:21 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:56:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:21 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi019.svjimj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-02-07T12:56:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:21 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:56:21.370 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:21 smithi172 ceph-mon[67636]: Reconfiguring daemon iscsi.foo.smithi019.svjimj on smithi019 2024-02-07T12:56:22.299 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:22 smithi019 ceph-mon[88137]: Health check failed: Upgrading daemon iscsi.foo.smithi019.svjimj on host smithi019 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-02-07T12:56:22.299 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:22 smithi019 ceph-mon[82918]: Health check failed: Upgrading daemon iscsi.foo.smithi019.svjimj on host smithi019 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-02-07T12:56:22.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:22 smithi172 ceph-mon[67636]: Health check failed: Upgrading daemon iscsi.foo.smithi019.svjimj on host smithi019 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-02-07T12:56:23.211 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:23 smithi019 ceph-mon[82918]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 652 B/s rd, 0 op/s 2024-02-07T12:56:23.211 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:23 smithi019 ceph-mon[88137]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 652 B/s rd, 0 op/s 2024-02-07T12:56:23.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:23 smithi172 ceph-mon[67636]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 652 B/s rd, 0 op/s 2024-02-07T12:56:24.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:56:23] "GET /metrics HTTP/1.1" 200 38407 "" "Prometheus/2.43.0" 2024-02-07T12:56:24.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:24 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:24.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:24 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:24.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:24 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-02-07T12:56:24.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:24 smithi019 ceph-mon[88137]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-02-07T12:56:24.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:24 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-02-07T12:56:24.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:24 smithi019 ceph-mon[88137]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-02-07T12:56:24.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:24 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:56:24.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:24 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/58708701' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-02-07T12:56:24.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:24 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/4172316114' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1075193736"}]: dispatch 2024-02-07T12:56:24.595 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:24 smithi019 ceph-mon[88137]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1075193736"}]: dispatch 2024-02-07T12:56:24.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:24 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:24.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:24 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:24.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:24 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-02-07T12:56:24.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:24 smithi019 ceph-mon[82918]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-02-07T12:56:24.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:24 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-02-07T12:56:24.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:24 smithi019 ceph-mon[82918]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-02-07T12:56:24.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:24 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:56:24.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:24 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/58708701' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-02-07T12:56:24.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:24 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/4172316114' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1075193736"}]: dispatch 2024-02-07T12:56:24.595 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:24 smithi019 ceph-mon[82918]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1075193736"}]: dispatch 2024-02-07T12:56:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:24 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:24 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:24 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-02-07T12:56:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:24 smithi172 ceph-mon[67636]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-02-07T12:56:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:24 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-02-07T12:56:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:24 smithi172 ceph-mon[67636]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-02-07T12:56:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:24 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:56:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:24 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/58708701' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-02-07T12:56:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:24 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/4172316114' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1075193736"}]: dispatch 2024-02-07T12:56:24.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:24 smithi172 ceph-mon[67636]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1075193736"}]: dispatch 2024-02-07T12:56:25.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:25 smithi172 ceph-mon[67636]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1075193736"}]': finished 2024-02-07T12:56:25.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:25 smithi172 ceph-mon[67636]: osdmap e147: 8 total, 8 up, 8 in 2024-02-07T12:56:25.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:25 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/3015008983' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/3127200756"}]: dispatch 2024-02-07T12:56:25.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:25 smithi172 ceph-mon[67636]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/3127200756"}]: dispatch 2024-02-07T12:56:25.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:25 smithi172 ceph-mon[67636]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 350 B/s rd, 0 op/s 2024-02-07T12:56:25.736 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:25 smithi019 ceph-mon[82918]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1075193736"}]': finished 2024-02-07T12:56:25.736 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:25 smithi019 ceph-mon[82918]: osdmap e147: 8 total, 8 up, 8 in 2024-02-07T12:56:25.736 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:25 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/3015008983' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/3127200756"}]: dispatch 2024-02-07T12:56:25.736 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:25 smithi019 ceph-mon[82918]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/3127200756"}]: dispatch 2024-02-07T12:56:25.736 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:25 smithi019 ceph-mon[82918]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 350 B/s rd, 0 op/s 2024-02-07T12:56:25.736 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:25 smithi019 ceph-mon[88137]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/1075193736"}]': finished 2024-02-07T12:56:25.737 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:25 smithi019 ceph-mon[88137]: osdmap e147: 8 total, 8 up, 8 in 2024-02-07T12:56:25.737 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:25 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/3015008983' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/3127200756"}]: dispatch 2024-02-07T12:56:25.737 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:25 smithi019 ceph-mon[88137]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/3127200756"}]: dispatch 2024-02-07T12:56:25.737 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:25 smithi019 ceph-mon[88137]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 350 B/s rd, 0 op/s 2024-02-07T12:56:26.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:26 smithi172 ceph-mon[67636]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/3127200756"}]': finished 2024-02-07T12:56:26.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:26 smithi172 ceph-mon[67636]: osdmap e148: 8 total, 8 up, 8 in 2024-02-07T12:56:26.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:26 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/2456398412' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/3738346934"}]: dispatch 2024-02-07T12:56:26.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:26 smithi172 ceph-mon[67636]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/3738346934"}]: dispatch 2024-02-07T12:56:26.660 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:26 smithi019 ceph-mon[82918]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/3127200756"}]': finished 2024-02-07T12:56:26.660 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:26 smithi019 ceph-mon[82918]: osdmap e148: 8 total, 8 up, 8 in 2024-02-07T12:56:26.660 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:26 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/2456398412' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/3738346934"}]: dispatch 2024-02-07T12:56:26.660 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:26 smithi019 ceph-mon[82918]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/3738346934"}]: dispatch 2024-02-07T12:56:26.661 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:26 smithi019 ceph-mon[88137]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/3127200756"}]': finished 2024-02-07T12:56:26.661 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:26 smithi019 ceph-mon[88137]: osdmap e148: 8 total, 8 up, 8 in 2024-02-07T12:56:26.661 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:26 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/2456398412' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/3738346934"}]: dispatch 2024-02-07T12:56:26.661 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:26 smithi019 ceph-mon[88137]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/3738346934"}]: dispatch 2024-02-07T12:56:27.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:27 smithi019 ceph-mon[88137]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/3738346934"}]': finished 2024-02-07T12:56:27.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:27 smithi019 ceph-mon[88137]: osdmap e149: 8 total, 8 up, 8 in 2024-02-07T12:56:27.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:27 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/3547434993' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3393948945"}]: dispatch 2024-02-07T12:56:27.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:27 smithi019 ceph-mon[88137]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-07T12:56:27.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:27 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:27.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:27 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:27.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:27 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:56:27.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:27 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:56:27.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:27 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:27.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:27 smithi019 ceph-mon[82918]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/3738346934"}]': finished 2024-02-07T12:56:27.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:27 smithi019 ceph-mon[82918]: osdmap e149: 8 total, 8 up, 8 in 2024-02-07T12:56:27.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:27 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/3547434993' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3393948945"}]: dispatch 2024-02-07T12:56:27.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:27 smithi019 ceph-mon[82918]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-07T12:56:27.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:27 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:27.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:27 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:27.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:27 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:56:27.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:27 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:56:27.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:27 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:27.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:27 smithi172 ceph-mon[67636]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6800/3738346934"}]': finished 2024-02-07T12:56:27.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:27 smithi172 ceph-mon[67636]: osdmap e149: 8 total, 8 up, 8 in 2024-02-07T12:56:27.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:27 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/3547434993' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3393948945"}]: dispatch 2024-02-07T12:56:27.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:27 smithi172 ceph-mon[67636]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-07T12:56:27.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:27 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:27.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:27 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:27.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:27 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:56:27.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:27 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:56:27.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:27 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:28.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:28 smithi019 ceph-mon[82918]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 480 B/s rd, 0 op/s 2024-02-07T12:56:28.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:28 smithi019 ceph-mon[82918]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-02-07T12:56:28.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:28 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/3547434993' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3393948945"}]': finished 2024-02-07T12:56:28.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:28 smithi019 ceph-mon[82918]: osdmap e150: 8 total, 8 up, 8 in 2024-02-07T12:56:28.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:28 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/1442052969' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/4027546410"}]: dispatch 2024-02-07T12:56:28.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:28 smithi019 ceph-mon[82918]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/4027546410"}]: dispatch 2024-02-07T12:56:28.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:28 smithi019 ceph-mon[88137]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 480 B/s rd, 0 op/s 2024-02-07T12:56:28.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:28 smithi019 ceph-mon[88137]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-02-07T12:56:28.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:28 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/3547434993' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3393948945"}]': finished 2024-02-07T12:56:28.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:28 smithi019 ceph-mon[88137]: osdmap e150: 8 total, 8 up, 8 in 2024-02-07T12:56:28.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:28 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/1442052969' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/4027546410"}]: dispatch 2024-02-07T12:56:28.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:28 smithi019 ceph-mon[88137]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/4027546410"}]: dispatch 2024-02-07T12:56:28.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:28 smithi172 ceph-mon[67636]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 480 B/s rd, 0 op/s 2024-02-07T12:56:28.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:28 smithi172 ceph-mon[67636]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-02-07T12:56:28.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:28 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/3547434993' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3393948945"}]': finished 2024-02-07T12:56:28.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:28 smithi172 ceph-mon[67636]: osdmap e150: 8 total, 8 up, 8 in 2024-02-07T12:56:28.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:28 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/1442052969' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/4027546410"}]: dispatch 2024-02-07T12:56:28.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:28 smithi172 ceph-mon[67636]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/4027546410"}]: dispatch 2024-02-07T12:56:29.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:29 smithi019 ceph-mon[88137]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/4027546410"}]': finished 2024-02-07T12:56:29.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:29 smithi019 ceph-mon[88137]: osdmap e151: 8 total, 8 up, 8 in 2024-02-07T12:56:29.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:29 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/1695424579' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3365303645"}]: dispatch 2024-02-07T12:56:29.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:29 smithi019 ceph-mon[82918]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/4027546410"}]': finished 2024-02-07T12:56:29.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:29 smithi019 ceph-mon[82918]: osdmap e151: 8 total, 8 up, 8 in 2024-02-07T12:56:29.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:29 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/1695424579' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3365303645"}]: dispatch 2024-02-07T12:56:29.868 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:29 smithi172 ceph-mon[67636]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/4027546410"}]': finished 2024-02-07T12:56:29.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:29 smithi172 ceph-mon[67636]: osdmap e151: 8 total, 8 up, 8 in 2024-02-07T12:56:29.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:29 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/1695424579' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3365303645"}]: dispatch 2024-02-07T12:56:30.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:30 smithi019 ceph-mon[88137]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-02-07T12:56:30.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:30 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/1695424579' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3365303645"}]': finished 2024-02-07T12:56:30.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:30 smithi019 ceph-mon[88137]: osdmap e152: 8 total, 8 up, 8 in 2024-02-07T12:56:30.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:30 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/1190442205' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3053386787"}]: dispatch 2024-02-07T12:56:30.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:30 smithi019 ceph-mon[88137]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3053386787"}]: dispatch 2024-02-07T12:56:30.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:30 smithi019 ceph-mon[82918]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-02-07T12:56:30.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:30 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/1695424579' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3365303645"}]': finished 2024-02-07T12:56:30.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:30 smithi019 ceph-mon[82918]: osdmap e152: 8 total, 8 up, 8 in 2024-02-07T12:56:30.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:30 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/1190442205' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3053386787"}]: dispatch 2024-02-07T12:56:30.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:30 smithi019 ceph-mon[82918]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3053386787"}]: dispatch 2024-02-07T12:56:30.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:30 smithi172 ceph-mon[67636]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 260 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-02-07T12:56:30.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:30 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/1695424579' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3365303645"}]': finished 2024-02-07T12:56:30.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:30 smithi172 ceph-mon[67636]: osdmap e152: 8 total, 8 up, 8 in 2024-02-07T12:56:30.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:30 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/1190442205' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3053386787"}]: dispatch 2024-02-07T12:56:30.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:30 smithi172 ceph-mon[67636]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3053386787"}]: dispatch 2024-02-07T12:56:31.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:31 smithi019 ceph-mon[82918]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3053386787"}]': finished 2024-02-07T12:56:31.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:31 smithi019 ceph-mon[82918]: osdmap e153: 8 total, 8 up, 8 in 2024-02-07T12:56:31.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:31 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/1574224559' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/10553872"}]: dispatch 2024-02-07T12:56:31.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:31 smithi019 ceph-mon[82918]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/10553872"}]: dispatch 2024-02-07T12:56:31.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:31 smithi019 ceph-mon[88137]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3053386787"}]': finished 2024-02-07T12:56:31.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:31 smithi019 ceph-mon[88137]: osdmap e153: 8 total, 8 up, 8 in 2024-02-07T12:56:31.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:31 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/1574224559' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/10553872"}]: dispatch 2024-02-07T12:56:31.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:31 smithi019 ceph-mon[88137]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/10553872"}]: dispatch 2024-02-07T12:56:31.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:31 smithi172 ceph-mon[67636]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/3053386787"}]': finished 2024-02-07T12:56:31.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:31 smithi172 ceph-mon[67636]: osdmap e153: 8 total, 8 up, 8 in 2024-02-07T12:56:31.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:31 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/1574224559' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/10553872"}]: dispatch 2024-02-07T12:56:31.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:31 smithi172 ceph-mon[67636]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/10553872"}]: dispatch 2024-02-07T12:56:32.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:32 smithi019 ceph-mon[82918]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:56:32.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:32 smithi019 ceph-mon[82918]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/10553872"}]': finished 2024-02-07T12:56:32.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:32 smithi019 ceph-mon[82918]: osdmap e154: 8 total, 8 up, 8 in 2024-02-07T12:56:32.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:32 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/1333895212' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/3127200756"}]: dispatch 2024-02-07T12:56:32.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:32 smithi019 ceph-mon[82918]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/3127200756"}]: dispatch 2024-02-07T12:56:32.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:32 smithi019 ceph-mon[88137]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:56:32.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:32 smithi019 ceph-mon[88137]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/10553872"}]': finished 2024-02-07T12:56:32.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:32 smithi019 ceph-mon[88137]: osdmap e154: 8 total, 8 up, 8 in 2024-02-07T12:56:32.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:32 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/1333895212' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/3127200756"}]: dispatch 2024-02-07T12:56:32.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:32 smithi019 ceph-mon[88137]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/3127200756"}]: dispatch 2024-02-07T12:56:32.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:32 smithi172 ceph-mon[67636]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:56:32.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:32 smithi172 ceph-mon[67636]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/10553872"}]': finished 2024-02-07T12:56:32.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:32 smithi172 ceph-mon[67636]: osdmap e154: 8 total, 8 up, 8 in 2024-02-07T12:56:32.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:32 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/1333895212' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/3127200756"}]: dispatch 2024-02-07T12:56:32.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:32 smithi172 ceph-mon[67636]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/3127200756"}]: dispatch 2024-02-07T12:56:33.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:33 smithi019 ceph-mon[88137]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/3127200756"}]': finished 2024-02-07T12:56:33.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:33 smithi019 ceph-mon[88137]: osdmap e155: 8 total, 8 up, 8 in 2024-02-07T12:56:33.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:33 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/933943617' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2381234619"}]: dispatch 2024-02-07T12:56:33.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:33 smithi019 ceph-mon[88137]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2381234619"}]: dispatch 2024-02-07T12:56:33.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:33 smithi019 ceph-mon[82918]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/3127200756"}]': finished 2024-02-07T12:56:33.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:33 smithi019 ceph-mon[82918]: osdmap e155: 8 total, 8 up, 8 in 2024-02-07T12:56:33.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:33 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/933943617' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2381234619"}]: dispatch 2024-02-07T12:56:33.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:33 smithi019 ceph-mon[82918]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2381234619"}]: dispatch 2024-02-07T12:56:33.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:33 smithi172 ceph-mon[67636]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/3127200756"}]': finished 2024-02-07T12:56:33.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:33 smithi172 ceph-mon[67636]: osdmap e155: 8 total, 8 up, 8 in 2024-02-07T12:56:33.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:33 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/933943617' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2381234619"}]: dispatch 2024-02-07T12:56:33.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:33 smithi172 ceph-mon[67636]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2381234619"}]: dispatch 2024-02-07T12:56:34.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:56:33] "GET /metrics HTTP/1.1" 200 38407 "" "Prometheus/2.43.0" 2024-02-07T12:56:34.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:34 smithi019 ceph-mon[88137]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:56:34.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:34 smithi019 ceph-mon[88137]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2381234619"}]': finished 2024-02-07T12:56:34.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:34 smithi019 ceph-mon[88137]: osdmap e156: 8 total, 8 up, 8 in 2024-02-07T12:56:34.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:34 smithi019 ceph-mon[88137]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:56:34.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:34 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/3187572555' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/4105936552"}]: dispatch 2024-02-07T12:56:34.845 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:34 smithi019 ceph-mon[88137]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/4105936552"}]: dispatch 2024-02-07T12:56:34.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:34 smithi019 ceph-mon[82918]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:56:34.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:34 smithi019 ceph-mon[82918]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2381234619"}]': finished 2024-02-07T12:56:34.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:34 smithi019 ceph-mon[82918]: osdmap e156: 8 total, 8 up, 8 in 2024-02-07T12:56:34.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:34 smithi019 ceph-mon[82918]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:56:34.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:34 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/3187572555' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/4105936552"}]: dispatch 2024-02-07T12:56:34.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:34 smithi019 ceph-mon[82918]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/4105936552"}]: dispatch 2024-02-07T12:56:34.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:34 smithi172 ceph-mon[67636]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:56:34.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:34 smithi172 ceph-mon[67636]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/2381234619"}]': finished 2024-02-07T12:56:34.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:34 smithi172 ceph-mon[67636]: osdmap e156: 8 total, 8 up, 8 in 2024-02-07T12:56:34.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:34 smithi172 ceph-mon[67636]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:56:34.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:34 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/3187572555' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/4105936552"}]: dispatch 2024-02-07T12:56:34.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:34 smithi172 ceph-mon[67636]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/4105936552"}]: dispatch 2024-02-07T12:56:35.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:35 smithi019 ceph-mon[88137]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/4105936552"}]': finished 2024-02-07T12:56:35.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:35 smithi019 ceph-mon[88137]: osdmap e157: 8 total, 8 up, 8 in 2024-02-07T12:56:35.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:35 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/2945302224' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/3738346934"}]: dispatch 2024-02-07T12:56:35.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:35 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:35.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:35 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:56:35.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:35 smithi019 ceph-mon[82918]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/4105936552"}]': finished 2024-02-07T12:56:35.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:35 smithi019 ceph-mon[82918]: osdmap e157: 8 total, 8 up, 8 in 2024-02-07T12:56:35.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:35 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/2945302224' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/3738346934"}]: dispatch 2024-02-07T12:56:35.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:35 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:35.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:35 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:56:35.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:35 smithi172 ceph-mon[67636]: from='client.? ' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:0/4105936552"}]': finished 2024-02-07T12:56:35.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:35 smithi172 ceph-mon[67636]: osdmap e157: 8 total, 8 up, 8 in 2024-02-07T12:56:35.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:35 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/2945302224' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/3738346934"}]: dispatch 2024-02-07T12:56:35.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:35 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:56:35.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:35 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:56:36.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:36 smithi019 ceph-mon[88137]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:56:36.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:36 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/2945302224' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/3738346934"}]': finished 2024-02-07T12:56:36.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:36 smithi019 ceph-mon[88137]: osdmap e158: 8 total, 8 up, 8 in 2024-02-07T12:56:36.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:36 smithi019 ceph-mon[82918]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:56:36.844 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:36 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/2945302224' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/3738346934"}]': finished 2024-02-07T12:56:36.845 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:36 smithi019 ceph-mon[82918]: osdmap e158: 8 total, 8 up, 8 in 2024-02-07T12:56:36.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:36 smithi172 ceph-mon[67636]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:56:36.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:36 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/2945302224' entity='client.iscsi.foo.smithi019.svjimj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.19:6801/3738346934"}]': finished 2024-02-07T12:56:36.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:36 smithi172 ceph-mon[67636]: osdmap e158: 8 total, 8 up, 8 in 2024-02-07T12:56:36.910 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:56:37.249 INFO:teuthology.orchestra.run.smithi019.stdout:"Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi019.svjimj on host smithi019 failed." 2024-02-07T12:56:37.562 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:37 smithi019 ceph-mon[88137]: from='client.54874 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:56:37.562 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:37 smithi019 ceph-mon[82918]: from='client.54874 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:56:37.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:37 smithi172 ceph-mon[67636]: from='client.54874 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:56:37.988 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch ps' 2024-02-07T12:56:38.593 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:38 smithi019 ceph-mon[88137]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:56:38.593 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:38 smithi019 ceph-mon[88137]: from='client.54880 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:56:38.593 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:38 smithi019 ceph-mon[82918]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:56:38.593 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:38 smithi019 ceph-mon[82918]: from='client.54880 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:56:38.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:38 smithi172 ceph-mon[67636]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-02-07T12:56:38.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:38 smithi172 ceph-mon[67636]: from='client.54880 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:56:40.080 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:56:40.081 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (10m) 13s ago 32m - - 0.25.0 c8568f914cd2 fec9255f35e7 2024-02-07T12:56:40.081 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (10m) 69s ago 32m 86.0M - 9.4.12 a0ea5e3c46d9 aaffac63bc28 2024-02-07T12:56:40.081 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 running (17s) 13s ago 31m - - 3.6 79957b372307 027fd98c8b91 2024-02-07T12:56:40.081 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283,8765 running (11m) 69s ago 37m 452M - 19.0.0-1252-g44458cce 79957b372307 5aa0c04fdfd7 2024-02-07T12:56:40.081 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (22m) 13s ago 38m - - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:56:40.081 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (8m) 13s ago 39m - 2048M 19.0.0-1252-g44458cce 79957b372307 8b1b0e21b1c5 2024-02-07T12:56:40.081 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (9m) 69s ago 37m 56.4M 2048M 19.0.0-1252-g44458cce 79957b372307 00f454bbac01 2024-02-07T12:56:40.081 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (8m) 13s ago 37m - 2048M 19.0.0-1252-g44458cce 79957b372307 6214003f6e08 2024-02-07T12:56:40.081 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (11m) 13s ago 33m - - 1.5.0 0da6a335fe13 c4bdcb1c4d49 2024-02-07T12:56:40.081 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (11m) 69s ago 33m 16.4M - 1.5.0 0da6a335fe13 04ea81f79d85 2024-02-07T12:56:40.081 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (6m) 13s ago 36m - 2993M 19.0.0-1252-g44458cce 79957b372307 78eddd34a903 2024-02-07T12:56:40.081 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (6m) 13s ago 36m - 2993M 19.0.0-1252-g44458cce 79957b372307 c37447e76c38 2024-02-07T12:56:40.081 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (5m) 13s ago 36m - 2993M 19.0.0-1252-g44458cce 79957b372307 90352897da85 2024-02-07T12:56:40.081 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (4m) 13s ago 35m - 2993M 19.0.0-1252-g44458cce 79957b372307 067cd6b3fd2f 2024-02-07T12:56:40.082 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (3m) 69s ago 35m 51.5M 3248M 19.0.0-1252-g44458cce 79957b372307 dbb47a8ad5ac 2024-02-07T12:56:40.082 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (3m) 69s ago 35m 48.4M 3248M 19.0.0-1252-g44458cce 79957b372307 d139d35fec57 2024-02-07T12:56:40.082 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (2m) 69s ago 34m 54.0M 3248M 19.0.0-1252-g44458cce 79957b372307 219cf25240d5 2024-02-07T12:56:40.082 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (2m) 69s ago 34m 50.0M 3248M 19.0.0-1252-g44458cce 79957b372307 75dab2b6606b 2024-02-07T12:56:40.082 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (11m) 69s ago 33m 71.7M - 2.43.0 a07b618ecd1d c9b607eea1be 2024-02-07T12:56:40.082 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (79s) 13s ago 32m - - 19.0.0-1252-g44458cce 79957b372307 3753defe9ced 2024-02-07T12:56:40.082 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (73s) 69s ago 32m 76.4M - 19.0.0-1252-g44458cce 79957b372307 6e879570d813 2024-02-07T12:56:40.841 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph versions' 2024-02-07T12:56:41.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:40 smithi019 ceph-mon[88137]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:56:41.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:40 smithi019 ceph-mon[82918]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:56:41.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:40 smithi172 ceph-mon[67636]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:56:42.211 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:41 smithi019 ceph-mon[88137]: from='client.54886 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:56:42.212 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:41 smithi019 ceph-mon[82918]: from='client.54886 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:56:42.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:41 smithi172 ceph-mon[67636]: from='client.54886 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:56:42.810 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:56:42.811 INFO:teuthology.orchestra.run.smithi019.stdout: "mon": { 2024-02-07T12:56:42.811 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 3 2024-02-07T12:56:42.811 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:56:42.811 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr": { 2024-02-07T12:56:42.811 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:56:42.811 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:56:42.811 INFO:teuthology.orchestra.run.smithi019.stdout: "osd": { 2024-02-07T12:56:42.811 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 8 2024-02-07T12:56:42.811 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:56:42.811 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw": { 2024-02-07T12:56:42.811 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:56:42.811 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:56:42.811 INFO:teuthology.orchestra.run.smithi019.stdout: "overall": { 2024-02-07T12:56:42.811 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 15 2024-02-07T12:56:42.812 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:56:42.812 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:56:43.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:42 smithi019 ceph-mon[82918]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 672 B/s rd, 0 op/s 2024-02-07T12:56:43.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:42 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/2417477586' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:43.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:42 smithi019 ceph-mon[88137]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 672 B/s rd, 0 op/s 2024-02-07T12:56:43.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:42 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/2417477586' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:43.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:42 smithi172 ceph-mon[67636]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 672 B/s rd, 0 op/s 2024-02-07T12:56:43.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:42 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/2417477586' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:56:43.468 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-02-07T12:56:44.079 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:56:43] "GET /metrics HTTP/1.1" 200 38486 "" "Prometheus/2.43.0" 2024-02-07T12:56:45.270 INFO:teuthology.orchestra.run.smithi019.stdout:wait for servicemap items w/ changing names to refresh 2024-02-07T12:56:45.270 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:44 smithi019 ceph-mon[88137]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:56:45.270 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:44 smithi019 ceph-mon[88137]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:56:45.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:44 smithi019 ceph-mon[82918]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:56:45.271 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:44 smithi019 ceph-mon[82918]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:56:45.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:44 smithi172 ceph-mon[67636]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:56:45.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:44 smithi172 ceph-mon[67636]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:56:45.911 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'sleep 60' 2024-02-07T12:56:47.121 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:46 smithi019 ceph-mon[82918]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:56:47.122 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:46 smithi019 ceph-mon[88137]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:56:47.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:46 smithi172 ceph-mon[67636]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-02-07T12:56:49.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:48 smithi019 ceph-mon[82918]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 883 B/s rd, 0 op/s 2024-02-07T12:56:49.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:48 smithi019 ceph-mon[88137]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 883 B/s rd, 0 op/s 2024-02-07T12:56:49.368 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:48 smithi172 ceph-mon[67636]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 883 B/s rd, 0 op/s 2024-02-07T12:56:51.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:50 smithi019 ceph-mon[82918]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:56:51.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:50 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:56:51.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:50 smithi019 ceph-mon[88137]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:56:51.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:50 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:56:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:50 smithi172 ceph-mon[67636]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:56:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:50 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:56:53.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:52 smithi019 ceph-mon[88137]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:56:53.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:52 smithi019 ceph-mon[82918]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:56:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:52 smithi172 ceph-mon[67636]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:56:54.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:56:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:56:53] "GET /metrics HTTP/1.1" 200 38489 "" "Prometheus/2.43.0" 2024-02-07T12:56:55.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:54 smithi019 ceph-mon[82918]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:56:55.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:54 smithi019 ceph-mon[82918]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:56:55.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:54 smithi019 ceph-mon[88137]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:56:55.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:54 smithi019 ceph-mon[88137]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:56:55.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:54 smithi172 ceph-mon[67636]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:56:55.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:54 smithi172 ceph-mon[67636]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:56:57.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:56 smithi019 ceph-mon[82918]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:56:57.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:56 smithi019 ceph-mon[88137]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:56:57.368 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:56 smithi172 ceph-mon[67636]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:56:59.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:56:58 smithi019 ceph-mon[88137]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:56:59.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:56:58 smithi019 ceph-mon[82918]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:56:59.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:56:58 smithi172 ceph-mon[67636]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:01.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:00 smithi019 ceph-mon[82918]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:01.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:00 smithi019 ceph-mon[88137]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:01.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:00 smithi172 ceph-mon[67636]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:03.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:02 smithi019 ceph-mon[82918]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:03.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:02 smithi019 ceph-mon[88137]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:03.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:02 smithi172 ceph-mon[67636]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:04.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:57:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:57:03] "GET /metrics HTTP/1.1" 200 38489 "" "Prometheus/2.43.0" 2024-02-07T12:57:05.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:04 smithi019 ceph-mon[88137]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:05.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:04 smithi019 ceph-mon[88137]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:57:05.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:04 smithi019 ceph-mon[82918]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:05.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:04 smithi019 ceph-mon[82918]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:57:05.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:04 smithi172 ceph-mon[67636]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:05.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:04 smithi172 ceph-mon[67636]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:57:06.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:06 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:57:06.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:06 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:57:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:06 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:57:07.264 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:07 smithi019 ceph-mon[88137]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-02-07T12:57:07.264 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:07 smithi019 ceph-mon[82918]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-02-07T12:57:07.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:07 smithi172 ceph-mon[67636]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-02-07T12:57:09.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:09 smithi019 ceph-mon[82918]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:09.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:09 smithi019 ceph-mon[88137]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:09 smithi172 ceph-mon[67636]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:11.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:11 smithi019 ceph-mon[82918]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:11.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:11 smithi019 ceph-mon[88137]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:11.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:11 smithi172 ceph-mon[67636]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:13.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:13 smithi019 ceph-mon[82918]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:13.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:13 smithi019 ceph-mon[88137]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:13.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:13 smithi172 ceph-mon[67636]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:14.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:57:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:57:13] "GET /metrics HTTP/1.1" 200 38487 "" "Prometheus/2.43.0" 2024-02-07T12:57:15.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:15 smithi019 ceph-mon[82918]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:15.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:15 smithi019 ceph-mon[82918]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:57:15.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:15 smithi019 ceph-mon[88137]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:15.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:15 smithi019 ceph-mon[88137]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:57:15.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:15 smithi172 ceph-mon[67636]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:15.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:15 smithi172 ceph-mon[67636]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:57:17.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:17 smithi019 ceph-mon[88137]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-02-07T12:57:17.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:17 smithi019 ceph-mon[82918]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-02-07T12:57:17.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:17 smithi172 ceph-mon[67636]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-02-07T12:57:19.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:19 smithi019 ceph-mon[82918]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:19.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:19 smithi019 ceph-mon[88137]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:19.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:19 smithi172 ceph-mon[67636]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:21.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:21 smithi019 ceph-mon[82918]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:21.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:21 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:57:21.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:21 smithi019 ceph-mon[88137]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:21.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:21 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:57:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:21 smithi172 ceph-mon[67636]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:21.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:21 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:57:23.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:23 smithi019 ceph-mon[82918]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:23.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:23 smithi019 ceph-mon[88137]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:23.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:23 smithi172 ceph-mon[67636]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:24.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:57:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:57:23] "GET /metrics HTTP/1.1" 200 38487 "" "Prometheus/2.43.0" 2024-02-07T12:57:24.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:24 smithi019 ceph-mon[82918]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:24.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:24 smithi019 ceph-mon[82918]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:57:24.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:24 smithi019 ceph-mon[88137]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:24.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:24 smithi019 ceph-mon[88137]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:57:24.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:24 smithi172 ceph-mon[67636]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:24.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:24 smithi172 ceph-mon[67636]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:57:27.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:26 smithi019 ceph-mon[82918]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:27.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:26 smithi019 ceph-mon[88137]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:27.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:26 smithi172 ceph-mon[67636]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:28.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:27 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:57:28.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:27 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:57:28.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:27 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:57:28.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:27 smithi019 ceph-mon[82918]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:57:28.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:27 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:57:28.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:27 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:57:28.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:27 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:57:28.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:27 smithi019 ceph-mon[88137]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:57:28.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:27 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:57:28.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:27 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:57:28.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:27 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:57:28.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:27 smithi172 ceph-mon[67636]: from='mgr.44106 ' entity='mgr.y' 2024-02-07T12:57:29.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:28 smithi019 ceph-mon[88137]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:29.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:28 smithi019 ceph-mon[82918]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:28 smithi172 ceph-mon[67636]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:31.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:30 smithi019 ceph-mon[82918]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:31.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:30 smithi019 ceph-mon[88137]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:31.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:30 smithi172 ceph-mon[67636]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:33.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:32 smithi019 ceph-mon[82918]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:33.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:32 smithi019 ceph-mon[88137]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:33.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:32 smithi172 ceph-mon[67636]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:34.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:57:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:57:33] "GET /metrics HTTP/1.1" 200 38487 "" "Prometheus/2.43.0" 2024-02-07T12:57:35.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:34 smithi019 ceph-mon[82918]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:35.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:34 smithi019 ceph-mon[82918]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:57:35.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:34 smithi019 ceph-mon[88137]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:35.345 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:34 smithi019 ceph-mon[88137]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:57:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:34 smithi172 ceph-mon[67636]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:35.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:34 smithi172 ceph-mon[67636]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:57:36.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:35 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:57:36.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:35 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:57:36.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:35 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:57:37.245 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:36 smithi019 ceph-mon[88137]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:37.246 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:36 smithi019 ceph-mon[82918]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:37.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:36 smithi172 ceph-mon[67636]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:39.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:38 smithi019 ceph-mon[82918]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:39.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:38 smithi019 ceph-mon[88137]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:39.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:38 smithi172 ceph-mon[67636]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:41.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:40 smithi019 ceph-mon[82918]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:41.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:40 smithi019 ceph-mon[88137]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:41.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:40 smithi172 ceph-mon[67636]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:43.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:42 smithi019 ceph-mon[82918]: pgmap v386: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:43.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:42 smithi019 ceph-mon[88137]: pgmap v386: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:43.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:42 smithi172 ceph-mon[67636]: pgmap v386: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:44.344 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:57:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:57:43] "GET /metrics HTTP/1.1" 200 38488 "" "Prometheus/2.43.0" 2024-02-07T12:57:45.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:44 smithi019 ceph-mon[82918]: pgmap v387: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:45.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:44 smithi019 ceph-mon[82918]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:57:45.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:44 smithi019 ceph-mon[88137]: pgmap v387: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:45.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:44 smithi019 ceph-mon[88137]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:57:45.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:44 smithi172 ceph-mon[67636]: pgmap v387: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:45.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:44 smithi172 ceph-mon[67636]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:57:47.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:47 smithi019 ceph-mon[82918]: pgmap v388: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:47.347 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:47 smithi019 ceph-mon[88137]: pgmap v388: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:47.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:47 smithi172 ceph-mon[67636]: pgmap v388: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:48.230 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch ps' 2024-02-07T12:57:49.207 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:49 smithi019 ceph-mon[88137]: pgmap v389: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:49.207 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:49 smithi019 ceph-mon[82918]: pgmap v389: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:49.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:49 smithi172 ceph-mon[67636]: pgmap v389: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:50.167 INFO:teuthology.orchestra.run.smithi019.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:57:50.168 INFO:teuthology.orchestra.run.smithi019.stdout:alertmanager.a smithi019 *:9093,9094 running (12m) 83s ago 33m - - 0.25.0 c8568f914cd2 fec9255f35e7 2024-02-07T12:57:50.168 INFO:teuthology.orchestra.run.smithi019.stdout:grafana.a smithi172 *:3000 running (11m) 2m ago 33m 86.0M - 9.4.12 a0ea5e3c46d9 aaffac63bc28 2024-02-07T12:57:50.168 INFO:teuthology.orchestra.run.smithi019.stdout:iscsi.foo.smithi019.svjimj smithi019 running (87s) 83s ago 33m - - 3.6 79957b372307 027fd98c8b91 2024-02-07T12:57:50.168 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.x smithi172 *:8443,9283,8765 running (13m) 2m ago 38m 452M - 19.0.0-1252-g44458cce 79957b372307 5aa0c04fdfd7 2024-02-07T12:57:50.168 INFO:teuthology.orchestra.run.smithi019.stdout:mgr.y smithi019 *:8443,9283,8765 running (23m) 83s ago 40m - - 19.0.0-1252-g44458cce 79957b372307 11bffc8ee7da 2024-02-07T12:57:50.168 INFO:teuthology.orchestra.run.smithi019.stdout:mon.a smithi019 running (9m) 83s ago 40m - 2048M 19.0.0-1252-g44458cce 79957b372307 8b1b0e21b1c5 2024-02-07T12:57:50.168 INFO:teuthology.orchestra.run.smithi019.stdout:mon.b smithi172 running (10m) 2m ago 38m 56.4M 2048M 19.0.0-1252-g44458cce 79957b372307 00f454bbac01 2024-02-07T12:57:50.168 INFO:teuthology.orchestra.run.smithi019.stdout:mon.c smithi019 running (9m) 83s ago 38m - 2048M 19.0.0-1252-g44458cce 79957b372307 6214003f6e08 2024-02-07T12:57:50.168 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.a smithi019 *:9100 running (12m) 83s ago 34m - - 1.5.0 0da6a335fe13 c4bdcb1c4d49 2024-02-07T12:57:50.168 INFO:teuthology.orchestra.run.smithi019.stdout:node-exporter.b smithi172 *:9100 running (12m) 2m ago 34m 16.4M - 1.5.0 0da6a335fe13 04ea81f79d85 2024-02-07T12:57:50.168 INFO:teuthology.orchestra.run.smithi019.stdout:osd.0 smithi019 running (7m) 83s ago 38m - 2993M 19.0.0-1252-g44458cce 79957b372307 78eddd34a903 2024-02-07T12:57:50.168 INFO:teuthology.orchestra.run.smithi019.stdout:osd.1 smithi019 running (7m) 83s ago 37m - 2993M 19.0.0-1252-g44458cce 79957b372307 c37447e76c38 2024-02-07T12:57:50.169 INFO:teuthology.orchestra.run.smithi019.stdout:osd.2 smithi019 running (6m) 83s ago 37m - 2993M 19.0.0-1252-g44458cce 79957b372307 90352897da85 2024-02-07T12:57:50.169 INFO:teuthology.orchestra.run.smithi019.stdout:osd.3 smithi019 running (5m) 83s ago 36m - 2993M 19.0.0-1252-g44458cce 79957b372307 067cd6b3fd2f 2024-02-07T12:57:50.169 INFO:teuthology.orchestra.run.smithi019.stdout:osd.4 smithi172 running (5m) 2m ago 36m 51.5M 3248M 19.0.0-1252-g44458cce 79957b372307 dbb47a8ad5ac 2024-02-07T12:57:50.169 INFO:teuthology.orchestra.run.smithi019.stdout:osd.5 smithi172 running (4m) 2m ago 36m 48.4M 3248M 19.0.0-1252-g44458cce 79957b372307 d139d35fec57 2024-02-07T12:57:50.169 INFO:teuthology.orchestra.run.smithi019.stdout:osd.6 smithi172 running (3m) 2m ago 35m 54.0M 3248M 19.0.0-1252-g44458cce 79957b372307 219cf25240d5 2024-02-07T12:57:50.169 INFO:teuthology.orchestra.run.smithi019.stdout:osd.7 smithi172 running (3m) 2m ago 35m 50.0M 3248M 19.0.0-1252-g44458cce 79957b372307 75dab2b6606b 2024-02-07T12:57:50.169 INFO:teuthology.orchestra.run.smithi019.stdout:prometheus.a smithi172 *:9095 running (12m) 2m ago 34m 71.7M - 2.43.0 a07b618ecd1d c9b607eea1be 2024-02-07T12:57:50.169 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi019.ciualv smithi019 *:8000 running (2m) 83s ago 33m - - 19.0.0-1252-g44458cce 79957b372307 3753defe9ced 2024-02-07T12:57:50.169 INFO:teuthology.orchestra.run.smithi019.stdout:rgw.foo.smithi172.wueguy smithi172 *:8000 running (2m) 2m ago 33m 76.4M - 19.0.0-1252-g44458cce 79957b372307 6e879570d813 2024-02-07T12:57:50.888 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph versions' 2024-02-07T12:57:51.202 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:51 smithi019 ceph-mon[88137]: pgmap v390: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:51.202 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:51 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:57:51.203 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:51 smithi019 ceph-mon[82918]: pgmap v390: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:51.203 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:51 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:57:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:51 smithi172 ceph-mon[67636]: pgmap v390: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:51.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:51 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:57:52.318 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:52 smithi019 ceph-mon[88137]: from='client.54898 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:57:52.319 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:52 smithi019 ceph-mon[82918]: from='client.54898 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:57:52.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:52 smithi172 ceph-mon[67636]: from='client.54898 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:57:52.973 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:57:52.973 INFO:teuthology.orchestra.run.smithi019.stdout: "mon": { 2024-02-07T12:57:52.973 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 3 2024-02-07T12:57:52.974 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:57:52.974 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr": { 2024-02-07T12:57:52.974 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:57:52.974 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:57:52.974 INFO:teuthology.orchestra.run.smithi019.stdout: "osd": { 2024-02-07T12:57:52.974 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 8 2024-02-07T12:57:52.974 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:57:52.974 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw": { 2024-02-07T12:57:52.974 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:57:52.974 INFO:teuthology.orchestra.run.smithi019.stdout: }, 2024-02-07T12:57:52.974 INFO:teuthology.orchestra.run.smithi019.stdout: "overall": { 2024-02-07T12:57:52.974 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 15 2024-02-07T12:57:52.974 INFO:teuthology.orchestra.run.smithi019.stdout: } 2024-02-07T12:57:52.974 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:57:53.279 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:53 smithi019 ceph-mon[88137]: pgmap v391: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:53.279 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:53 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/76938531' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:57:53.279 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:53 smithi019 ceph-mon[82918]: pgmap v391: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:53.280 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:53 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/76938531' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:57:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:53 smithi172 ceph-mon[67636]: pgmap v391: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:53.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:53 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/76938531' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:57:53.611 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch upgrade status' 2024-02-07T12:57:54.223 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:57:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:57:53] "GET /metrics HTTP/1.1" 200 38488 "" "Prometheus/2.43.0" 2024-02-07T12:57:55.150 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:55 smithi019 ceph-mon[82918]: pgmap v392: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:55.150 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:55 smithi019 ceph-mon[82918]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:57:55.150 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:55 smithi019 ceph-mon[88137]: pgmap v392: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:55.150 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:55 smithi019 ceph-mon[88137]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:57:55.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:55 smithi172 ceph-mon[67636]: pgmap v392: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:57:55.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:55 smithi172 ceph-mon[67636]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:57:55.642 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:57:55.642 INFO:teuthology.orchestra.run.smithi019.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:57:55.642 INFO:teuthology.orchestra.run.smithi019.stdout: "in_progress": true, 2024-02-07T12:57:55.642 INFO:teuthology.orchestra.run.smithi019.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-07T12:57:55.642 INFO:teuthology.orchestra.run.smithi019.stdout: "services_complete": [ 2024-02-07T12:57:55.642 INFO:teuthology.orchestra.run.smithi019.stdout: "osd", 2024-02-07T12:57:55.642 INFO:teuthology.orchestra.run.smithi019.stdout: "mgr", 2024-02-07T12:57:55.642 INFO:teuthology.orchestra.run.smithi019.stdout: "rgw", 2024-02-07T12:57:55.642 INFO:teuthology.orchestra.run.smithi019.stdout: "iscsi", 2024-02-07T12:57:55.642 INFO:teuthology.orchestra.run.smithi019.stdout: "mon" 2024-02-07T12:57:55.643 INFO:teuthology.orchestra.run.smithi019.stdout: ], 2024-02-07T12:57:55.643 INFO:teuthology.orchestra.run.smithi019.stdout: "progress": "16/21 daemons upgraded", 2024-02-07T12:57:55.643 INFO:teuthology.orchestra.run.smithi019.stdout: "message": "Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi019.svjimj on host smithi019 failed.", 2024-02-07T12:57:55.643 INFO:teuthology.orchestra.run.smithi019.stdout: "is_paused": true 2024-02-07T12:57:55.643 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:57:56.395 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph health detail' 2024-02-07T12:57:57.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:57 smithi019 ceph-mon[88137]: pgmap v393: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:57.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:57 smithi019 ceph-mon[88137]: from='client.44970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:57:57.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:57 smithi019 ceph-mon[82918]: pgmap v393: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:57.345 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:57 smithi019 ceph-mon[82918]: from='client.44970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:57:57.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:57 smithi172 ceph-mon[67636]: pgmap v393: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:57.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:57 smithi172 ceph-mon[67636]: from='client.44970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:57:58.421 INFO:teuthology.orchestra.run.smithi019.stdout:HEALTH_WARN Upgrading daemon iscsi.foo.smithi019.svjimj on host smithi019 failed. 2024-02-07T12:57:58.421 INFO:teuthology.orchestra.run.smithi019.stdout:[WRN] UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi019.svjimj on host smithi019 failed. 2024-02-07T12:57:58.421 INFO:teuthology.orchestra.run.smithi019.stdout: Upgrade daemon: iscsi.foo.smithi019.svjimj: cephadm exited with an error code: 1, stderr: Redeploy daemon iscsi.foo.smithi019.svjimj ... 2024-02-07T12:57:58.421 INFO:teuthology.orchestra.run.smithi019.stdout:Creating ceph-iscsi config... 2024-02-07T12:57:58.421 INFO:teuthology.orchestra.run.smithi019.stdout:Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/iscsi-gateway.cfg 2024-02-07T12:57:58.421 INFO:teuthology.orchestra.run.smithi019.stdout:Write file: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/iscsi.foo.smithi019.svjimj/tcmu-runner-entrypoint.sh 2024-02-07T12:57:58.421 INFO:teuthology.orchestra.run.smithi019.stdout:Failed to trim old cgroups /sys/fs/cgroup/system.slice/system-ceph\x2dced2783c\x2dc5b2\x2d11ee\x2d95b6\x2d87774f69a715.slice/ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service 2024-02-07T12:57:58.421 INFO:teuthology.orchestra.run.smithi019.stdout:Non-zero exit code 1 from systemctl start ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj 2024-02-07T12:57:58.421 INFO:teuthology.orchestra.run.smithi019.stdout:systemctl: stderr Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:57:58.422 INFO:teuthology.orchestra.run.smithi019.stdout:systemctl: stderr See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:57:58.422 INFO:teuthology.orchestra.run.smithi019.stdout:Traceback (most recent call last): 2024-02-07T12:57:58.422 INFO:teuthology.orchestra.run.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-02-07T12:57:58.422 INFO:teuthology.orchestra.run.smithi019.stdout: return _run_code(code, main_globals, None, 2024-02-07T12:57:58.422 INFO:teuthology.orchestra.run.smithi019.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-02-07T12:57:58.422 INFO:teuthology.orchestra.run.smithi019.stdout: exec(code, run_globals) 2024-02-07T12:57:58.422 INFO:teuthology.orchestra.run.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5520, in 2024-02-07T12:57:58.422 INFO:teuthology.orchestra.run.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 5508, in main 2024-02-07T12:57:58.422 INFO:teuthology.orchestra.run.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3038, in command_deploy_from 2024-02-07T12:57:58.422 INFO:teuthology.orchestra.run.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3073, in _common_deploy 2024-02-07T12:57:58.422 INFO:teuthology.orchestra.run.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 3093, in _deploy_daemon_container 2024-02-07T12:57:58.422 INFO:teuthology.orchestra.run.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1029, in deploy_daemon 2024-02-07T12:57:58.422 INFO:teuthology.orchestra.run.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/__main__.py", line 1165, in deploy_daemon_units 2024-02-07T12:57:58.422 INFO:teuthology.orchestra.run.smithi019.stdout: File "/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/cephadm.5dbbc80103938c9fd7411408ec8dc25a6df1c83bdbf3723d92b57af1b1ae981c/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-02-07T12:57:58.422 INFO:teuthology.orchestra.run.smithi019.stdout:RuntimeError: Failed command: systemctl start ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj: Job for ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service failed because the control process exited with error code. 2024-02-07T12:57:58.423 INFO:teuthology.orchestra.run.smithi019.stdout:See "systemctl status ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" and "journalctl -xeu ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@iscsi.foo.smithi019.svjimj.service" for details. 2024-02-07T12:57:59.034 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-02-07T12:57:59.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:59 smithi019 ceph-mon[88137]: pgmap v394: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:59.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:57:59 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/3785484594' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:57:59.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:59 smithi019 ceph-mon[82918]: pgmap v394: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:59.341 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:57:59 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/3785484594' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:57:59.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:59 smithi172 ceph-mon[67636]: pgmap v394: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:57:59.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:57:59 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/3785484594' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:58:01.124 INFO:teuthology.orchestra.run.smithi019.stdout:true 2024-02-07T12:58:01.124 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:58:01 smithi019 ceph-mon[82918]: pgmap v395: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:58:01.124 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:58:01 smithi019 ceph-mon[88137]: pgmap v395: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:58:01.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:58:01 smithi172 ceph-mon[67636]: pgmap v395: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:58:01.776 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-02-07T12:58:02.340 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:58:02 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/1705711907' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:58:02.341 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:58:02 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/1705711907' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:58:02.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:58:02 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/1705711907' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:58:03.322 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:58:03 smithi019 ceph-mon[82918]: pgmap v396: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:58:03.322 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:58:03 smithi019 ceph-mon[88137]: pgmap v396: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:58:03.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:58:03 smithi172 ceph-mon[67636]: pgmap v396: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:58:03.826 INFO:teuthology.orchestra.run.smithi019.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)" 2024-02-07T12:58:04.094 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:58:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:58:03] "GET /metrics HTTP/1.1" 200 38488 "" "Prometheus/2.43.0" 2024-02-07T12:58:04.094 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:58:04 smithi019 ceph-mon[88137]: pgmap v397: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:58:04.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:58:04 smithi019 ceph-mon[88137]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:58:04.095 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:58:04 smithi019 ceph-mon[88137]: from='client.? 172.21.15.19:0/1161935831' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:58:04.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:58:04 smithi019 ceph-mon[82918]: pgmap v397: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:58:04.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:58:04 smithi019 ceph-mon[82918]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:58:04.095 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:58:04 smithi019 ceph-mon[82918]: from='client.? 172.21.15.19:0/1161935831' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:58:04.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:58:04 smithi172 ceph-mon[67636]: pgmap v397: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:58:04.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:58:04 smithi172 ceph-mon[67636]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:58:04.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:58:04 smithi172 ceph-mon[67636]: from='client.? 172.21.15.19:0/1161935831' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:58:04.408 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch ls | grep '"'"'^osd '"'"'' 2024-02-07T12:58:05.989 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:58:05 smithi019 ceph-mon[88137]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:58:05.989 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:58:05 smithi019 ceph-mon[82918]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:58:06.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:58:05 smithi172 ceph-mon[67636]: from='mgr.44106 172.21.15.19:0/2638208467' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:58:06.439 INFO:teuthology.orchestra.run.smithi019.stdout:osd 8 2m ago - 2024-02-07T12:58:07.118 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-02-07T12:58:07.128 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi019.front.sepia.ceph.com 2024-02-07T12:58:07.128 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- bash -c 'ceph orch upgrade ls' 2024-02-07T12:58:07.156 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:58:06 smithi019 ceph-mon[88137]: pgmap v398: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:58:07.156 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:58:06 smithi019 ceph-mon[82918]: pgmap v398: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:58:07.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:58:06 smithi172 ceph-mon[67636]: pgmap v398: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:58:08.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:58:07 smithi019 ceph-mon[82918]: from='client.44994 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:58:08.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:58:07 smithi019 ceph-mon[88137]: from='client.44994 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:58:08.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:58:07 smithi172 ceph-mon[67636]: from='client.44994 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:58:09.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:58:08 smithi019 ceph-mon[88137]: pgmap v399: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:58:09.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:58:08 smithi019 ceph-mon[82918]: pgmap v399: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:58:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:58:08 smithi172 ceph-mon[67636]: pgmap v399: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:58:09.932 INFO:teuthology.orchestra.run.smithi019.stdout:{ 2024-02-07T12:58:09.932 INFO:teuthology.orchestra.run.smithi019.stdout: "image": "quay.io/ceph/ceph", 2024-02-07T12:58:09.932 INFO:teuthology.orchestra.run.smithi019.stdout: "registry": "quay.io", 2024-02-07T12:58:09.932 INFO:teuthology.orchestra.run.smithi019.stdout: "bare_image": "ceph/ceph", 2024-02-07T12:58:09.932 INFO:teuthology.orchestra.run.smithi019.stdout: "versions": [] 2024-02-07T12:58:09.933 INFO:teuthology.orchestra.run.smithi019.stdout:} 2024-02-07T12:58:10.538 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0' 2024-02-07T12:58:11.182 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:58:10 smithi019 ceph-mon[82918]: pgmap v400: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:58:11.182 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:58:10 smithi019 ceph-mon[82918]: from='client.45000 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:58:11.182 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:58:10 smithi019 ceph-mon[88137]: pgmap v400: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:58:11.182 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:58:10 smithi019 ceph-mon[88137]: from='client.45000 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:58:11.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:58:10 smithi172 ceph-mon[67636]: pgmap v400: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:58:11.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:58:10 smithi172 ceph-mon[67636]: from='client.45000 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:58:13.112 INFO:teuthology.orchestra.run.smithi019.stdout: "16.2.0", 2024-02-07T12:58:13.344 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:58:13 smithi019 ceph-mon[88137]: pgmap v401: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:58:13.344 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:58:12 smithi019 ceph-mon[82918]: pgmap v401: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:58:13.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:58:12 smithi172 ceph-mon[67636]: pgmap v401: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:58:13.989 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2' 2024-02-07T12:58:14.307 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:58:14 smithi019 ceph-mon[88137]: from='client.54946 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:58:14.308 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:58:14 smithi019 ceph-mon[82918]: from='client.54946 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:58:14.308 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:58:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: ::ffff:172.21.15.172 - - [07/Feb/2024:12:58:13] "GET /metrics HTTP/1.1" 200 38485 "" "Prometheus/2.43.0" 2024-02-07T12:58:14.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:58:14 smithi172 ceph-mon[67636]: from='client.54946 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:58:15.274 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:58:15 smithi019 ceph-mon[88137]: pgmap v402: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:58:15.274 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:58:15 smithi019 ceph-mon[88137]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:58:15.274 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:58:15 smithi019 ceph-mon[82918]: pgmap v402: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:58:15.274 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:58:15 smithi019 ceph-mon[82918]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:58:15.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:58:15 smithi172 ceph-mon[67636]: pgmap v402: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-02-07T12:58:15.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:58:15 smithi172 ceph-mon[67636]: from='client.44847 -' entity='client.iscsi.foo.smithi019.svjimj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-02-07T12:58:16.592 INFO:teuthology.orchestra.run.smithi019.stdout: "v16.2.2", 2024-02-07T12:58:16.592 INFO:teuthology.orchestra.run.smithi019.stdout: "v16.2.2-20210505", 2024-02-07T12:58:17.173 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:58:17 smithi019 ceph-mon[88137]: pgmap v403: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:58:17.173 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:58:17 smithi019 ceph-mon[88137]: from='client.54952 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:58:17.173 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:58:17 smithi019 ceph-mon[82918]: pgmap v403: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:58:17.173 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:58:17 smithi019 ceph-mon[82918]: from='client.54952 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:58:17.202 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-02-07T12:58:17.212 INFO:tasks.cephadm:Teardown begin 2024-02-07T12:58:17.212 DEBUG:teuthology.orchestra.run.smithi019:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-02-07T12:58:17.245 DEBUG:teuthology.orchestra.run.smithi172:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-02-07T12:58:17.274 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:58:17 smithi172 ceph-mon[67636]: pgmap v403: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-02-07T12:58:17.275 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:58:17 smithi172 ceph-mon[67636]: from='client.54952 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:58:17.280 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-02-07T12:58:17.280 DEBUG:teuthology.orchestra.run.smithi019:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-02-07T12:58:17.301 DEBUG:teuthology.orchestra.run.smithi172:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-02-07T12:58:17.337 INFO:tasks.cephadm:Stopping all daemons... 2024-02-07T12:58:17.337 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-02-07T12:58:17.337 DEBUG:teuthology.orchestra.run.smithi019:> sudo systemctl stop ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mon.a 2024-02-07T12:58:17.594 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:58:17 smithi019 systemd[1]: Stopping Ceph mon.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:58:18.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:58:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-a[82914]: 2024-02-07T12:58:17.640+0000 7fbbf6a81700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:58:18.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:58:17 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-a[82914]: 2024-02-07T12:58:17.640+0000 7fbbf6a81700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-02-07T12:58:18.094 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:58:17 smithi019 podman[133983]: 2024-02-07 12:58:17.825873082 +0000 UTC m=+0.288575938 container died 8b1b0e21b1c548070914ac24243372f162013338f674502ce47637e51c54119d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-a, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, io.buildah.version=1.33.2) 2024-02-07T12:58:18.486 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:58:18 smithi019 podman[133983]: 2024-02-07 12:58:18.191793595 +0000 UTC m=+0.654496437 container cleanup 8b1b0e21b1c548070914ac24243372f162013338f674502ce47637e51c54119d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-a, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:58:18.486 INFO:journalctl@ceph.mon.a.smithi019.stdout:Feb 07 12:58:18 smithi019 bash[133983]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-a 2024-02-07T12:58:18.610 DEBUG:teuthology.orchestra.run.smithi019:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mon.a.service' 2024-02-07T12:58:18.665 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-07T12:58:18.665 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-02-07T12:58:18.665 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-02-07T12:58:18.665 DEBUG:teuthology.orchestra.run.smithi019:> sudo systemctl stop ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mon.c 2024-02-07T12:58:18.844 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:58:18 smithi019 systemd[1]: Stopping Ceph mon.c for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:58:19.298 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:58:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-c[88133]: 2024-02-07T12:58:18.970+0000 7ff1b8fa1700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:58:19.298 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:58:18 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-c[88133]: 2024-02-07T12:58:18.970+0000 7ff1b8fa1700 -1 mon.c@1(peon) e4 *** Got Signal Terminated *** 2024-02-07T12:58:19.298 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:58:19 smithi019 podman[134106]: 2024-02-07 12:58:19.012744085 +0000 UTC m=+0.155772490 container died 6214003f6e08b6bc4e7871cf9b636365217102d9b7431aadb7f35f80718c4d9d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-c, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True, org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-02-07T12:58:19.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:58:19 smithi019 podman[134106]: 2024-02-07 12:58:19.395988096 +0000 UTC m=+0.539016506 container cleanup 6214003f6e08b6bc4e7871cf9b636365217102d9b7431aadb7f35f80718c4d9d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-c, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240131, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, CEPH_POINT_RELEASE=, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:58:19.594 INFO:journalctl@ceph.mon.c.smithi019.stdout:Feb 07 12:58:19 smithi019 bash[134106]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-c 2024-02-07T12:58:19.823 DEBUG:teuthology.orchestra.run.smithi019:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mon.c.service' 2024-02-07T12:58:19.859 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-07T12:58:19.859 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-02-07T12:58:19.859 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-02-07T12:58:19.860 DEBUG:teuthology.orchestra.run.smithi172:> sudo systemctl stop ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mon.b 2024-02-07T12:58:20.184 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:58:19 smithi172 systemd[1]: Stopping Ceph mon.b for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:58:20.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:58:20 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-b[67632]: 2024-02-07T12:58:20.181+0000 7fb65ec75700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:58:20.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:58:20 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-b[67632]: 2024-02-07T12:58:20.181+0000 7fb65ec75700 -1 mon.b@2(peon) e4 *** Got Signal Terminated *** 2024-02-07T12:58:20.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:58:20 smithi172 podman[100428]: 2024-02-07 12:58:20.201825758 +0000 UTC m=+0.142748202 container died 00f454bbac01aa3cd5db09cb8e50ae2a5d08c2577d65b3af7715ee8d4d7ef6ac (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-b, ceph=True, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-02-07T12:58:20.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:58:20 smithi172 podman[100428]: 2024-02-07 12:58:20.69001434 +0000 UTC m=+0.630936836 container cleanup 00f454bbac01aa3cd5db09cb8e50ae2a5d08c2577d65b3af7715ee8d4d7ef6ac (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-b, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.build-date=20240131, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:58:20.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Feb 07 12:58:20 smithi172 bash[100428]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mon-b 2024-02-07T12:58:21.070 DEBUG:teuthology.orchestra.run.smithi172:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mon.b.service' 2024-02-07T12:58:21.132 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-07T12:58:21.132 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-02-07T12:58:21.132 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-02-07T12:58:21.132 DEBUG:teuthology.orchestra.run.smithi019:> sudo systemctl stop ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mgr.y 2024-02-07T12:58:21.435 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:58:21 smithi019 systemd[1]: Stopping Ceph mgr.y for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:58:21.769 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:58:21 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:58:21.433+0000 7fb78b6fb700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:58:21.770 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:58:21 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y[61872]: 2024-02-07T12:58:21.433+0000 7fb78b6fb700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-02-07T12:58:21.770 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:58:21 smithi019 podman[134227]: 2024-02-07 12:58:21.464911201 +0000 UTC m=+0.142711860 container died 11bffc8ee7dad2dcb75654595e3891267fe8186f218283242da10338be349402 (image=quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, org.label-schema.build-date=20240131, io.buildah.version=1.33.2) 2024-02-07T12:58:22.087 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:58:21 smithi019 podman[134227]: 2024-02-07 12:58:21.842752133 +0000 UTC m=+0.520552792 container cleanup 11bffc8ee7dad2dcb75654595e3891267fe8186f218283242da10338be349402 (image=quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:58:22.087 INFO:journalctl@ceph.mgr.y.smithi019.stdout:Feb 07 12:58:21 smithi019 bash[134227]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-y 2024-02-07T12:58:22.254 DEBUG:teuthology.orchestra.run.smithi019:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mgr.y.service' 2024-02-07T12:58:22.290 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-07T12:58:22.290 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-02-07T12:58:22.290 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-02-07T12:58:22.290 DEBUG:teuthology.orchestra.run.smithi172:> sudo systemctl stop ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mgr.x 2024-02-07T12:58:22.619 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:58:22 smithi172 systemd[1]: Stopping Ceph mgr.x for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:58:22.914 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:58:22 smithi172 podman[100549]: 2024-02-07 12:58:22.634128522 +0000 UTC m=+0.150325108 container died 5aa0c04fdfd76ef55235a01bce705de9af99c7dace7efe18bd820d6f2c023853 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-02-07T12:58:23.256 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:58:23 smithi172 podman[100549]: 2024-02-07 12:58:23.070300748 +0000 UTC m=+0.586497305 container cleanup 5aa0c04fdfd76ef55235a01bce705de9af99c7dace7efe18bd820d6f2c023853 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240131) 2024-02-07T12:58:23.257 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:58:23 smithi172 bash[100549]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x 2024-02-07T12:58:23.482 DEBUG:teuthology.orchestra.run.smithi172:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mgr.x.service' 2024-02-07T12:58:23.507 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:58:23 smithi172 podman[100563]: 2024-02-07 12:58:23.255094308 +0000 UTC m=+0.616970917 container remove 5aa0c04fdfd76ef55235a01bce705de9af99c7dace7efe18bd820d6f2c023853 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-02-07T12:58:23.508 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:58:23 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-02-07T12:58:23.508 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:58:23 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mgr.x.service: Failed with result 'exit-code'. 2024-02-07T12:58:23.508 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:58:23 smithi172 systemd[1]: Stopped Ceph mgr.x for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:58:23.508 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Feb 07 12:58:23 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@mgr.x.service: Consumed 23.016s CPU time. 2024-02-07T12:58:23.519 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-07T12:58:23.519 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-02-07T12:58:23.520 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-02-07T12:58:23.520 DEBUG:teuthology.orchestra.run.smithi019:> sudo systemctl stop ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.0 2024-02-07T12:58:23.806 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:58:23 smithi019 systemd[1]: Stopping Ceph osd.0 for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:58:24.094 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:58:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0[96125]: 2024-02-07T12:58:23.803+0000 7f6bbf5a6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:58:24.094 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:58:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0[96125]: 2024-02-07T12:58:23.803+0000 7f6bbf5a6700 -1 osd.0 158 *** Got signal Terminated *** 2024-02-07T12:58:24.094 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:58:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0[96125]: 2024-02-07T12:58:23.803+0000 7f6bbf5a6700 -1 osd.0 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:58:25.869 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:58:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:58:25.377Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.19:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.19:8765: connect: connection refused" 2024-02-07T12:58:25.869 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:58:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:58:25.378Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.19:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.19:8765: connect: connection refused" 2024-02-07T12:58:25.869 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:58:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:58:25.381Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.19:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.19:8765: connect: connection refused" 2024-02-07T12:58:25.869 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:58:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:58:25.381Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.19:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.19:8765: connect: connection refused" 2024-02-07T12:58:29.594 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:58:29 smithi019 podman[134348]: 2024-02-07 12:58:29.338935299 +0000 UTC m=+5.627780524 container died 78eddd34a903d9964edb15fe097be997606a173da04078f886cfcf6414cf48db (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.schema-version=1.0) 2024-02-07T12:58:29.965 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:58:29 smithi019 podman[134348]: 2024-02-07 12:58:29.738296823 +0000 UTC m=+6.027142042 container cleanup 78eddd34a903d9964edb15fe097be997606a173da04078f886cfcf6414cf48db (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0, GIT_BRANCH=HEAD, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-02-07T12:58:29.966 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:58:29 smithi019 bash[134348]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0 2024-02-07T12:58:30.268 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:58:29 smithi019 podman[134366]: 2024-02-07 12:58:29.964628584 +0000 UTC m=+0.617051111 container remove 78eddd34a903d9964edb15fe097be997606a173da04078f886cfcf6414cf48db (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131, GIT_CLEAN=True, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:58:30.520 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:58:30 smithi019 podman[134456]: 2024-02-07 12:58:30.170492305 +0000 UTC m=+0.018265932 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:58:30.520 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:58:30 smithi019 podman[134456]: 2024-02-07 12:58:30.307410428 +0000 UTC m=+0.155184047 container create f24e7683d9361724580fbc3ab1e33faadb94c4796cea7261aac36333d76601c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0-deactivate, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.2, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:58:30.821 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:58:30 smithi019 podman[134456]: 2024-02-07 12:58:30.519100233 +0000 UTC m=+0.366873855 container init f24e7683d9361724580fbc3ab1e33faadb94c4796cea7261aac36333d76601c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0-deactivate, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:58:30.821 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:58:30 smithi019 podman[134456]: 2024-02-07 12:58:30.52228244 +0000 UTC m=+0.370056060 container start f24e7683d9361724580fbc3ab1e33faadb94c4796cea7261aac36333d76601c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, io.buildah.version=1.33.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD) 2024-02-07T12:58:30.821 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:58:30 smithi019 podman[134456]: 2024-02-07 12:58:30.601487004 +0000 UTC m=+0.449260641 container attach f24e7683d9361724580fbc3ab1e33faadb94c4796cea7261aac36333d76601c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0-deactivate, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-02-07T12:58:31.094 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:58:30 smithi019 podman[134456]: 2024-02-07 12:58:30.8202269 +0000 UTC m=+0.668000521 container died f24e7683d9361724580fbc3ab1e33faadb94c4796cea7261aac36333d76601c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0-deactivate, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:58:31.391 INFO:journalctl@ceph.osd.0.smithi019.stdout:Feb 07 12:58:31 smithi019 podman[134482]: 2024-02-07 12:58:31.371650386 +0000 UTC m=+0.546212775 container remove f24e7683d9361724580fbc3ab1e33faadb94c4796cea7261aac36333d76601c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:58:31.396 DEBUG:teuthology.orchestra.run.smithi019:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.0.service' 2024-02-07T12:58:31.441 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-07T12:58:31.441 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-02-07T12:58:31.441 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-02-07T12:58:31.442 DEBUG:teuthology.orchestra.run.smithi019:> sudo systemctl stop ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.1 2024-02-07T12:58:31.659 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:58:31 smithi019 systemd[1]: Stopping Ceph osd.1 for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:58:32.094 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:58:31 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1[101137]: 2024-02-07T12:58:31.773+0000 7f1b15ee6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:58:32.094 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:58:31 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1[101137]: 2024-02-07T12:58:31.773+0000 7f1b15ee6700 -1 osd.1 158 *** Got signal Terminated *** 2024-02-07T12:58:32.094 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:58:31 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1[101137]: 2024-02-07T12:58:31.773+0000 7f1b15ee6700 -1 osd.1 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:58:33.844 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:58:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:58:33.553Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi019.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.19:8443: connect: connection refused" 2024-02-07T12:58:33.844 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:58:33 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:58:33.554Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi172.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.172:8443: connect: connection refused" 2024-02-07T12:58:37.594 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:58:37 smithi019 podman[134563]: 2024-02-07 12:58:37.308282419 +0000 UTC m=+5.665304051 container died c37447e76c38ce3e6d1d82848733c1563d4ce9dbfcd2868b84d5992d49912101 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-02-07T12:58:38.035 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:58:37 smithi019 podman[134563]: 2024-02-07 12:58:37.783532378 +0000 UTC m=+6.140554001 container cleanup c37447e76c38ce3e6d1d82848733c1563d4ce9dbfcd2868b84d5992d49912101 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux ) 2024-02-07T12:58:38.036 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:58:37 smithi019 bash[134563]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1 2024-02-07T12:58:38.344 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:58:38 smithi019 podman[134589]: 2024-02-07 12:58:38.034797236 +0000 UTC m=+0.726937683 container remove c37447e76c38ce3e6d1d82848733c1563d4ce9dbfcd2868b84d5992d49912101 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1, org.label-schema.build-date=20240131, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , io.buildah.version=1.33.2, ceph=True, org.label-schema.schema-version=1.0) 2024-02-07T12:58:38.683 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:58:38 smithi019 podman[134686]: 2024-02-07 12:58:38.355059005 +0000 UTC m=+0.111045038 container create 200c4768291ea66ea8aae6c91079ca6fc17e9ffae609ee18c80a2b205ab15dec (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-02-07T12:58:38.684 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:58:38 smithi019 podman[134686]: 2024-02-07 12:58:38.262690795 +0000 UTC m=+0.018676838 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:58:38.684 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:58:38 smithi019 podman[134686]: 2024-02-07 12:58:38.582451908 +0000 UTC m=+0.338437942 container init 200c4768291ea66ea8aae6c91079ca6fc17e9ffae609ee18c80a2b205ab15dec (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1-deactivate, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240131) 2024-02-07T12:58:38.684 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:58:38 smithi019 podman[134686]: 2024-02-07 12:58:38.585398832 +0000 UTC m=+0.341384865 container start 200c4768291ea66ea8aae6c91079ca6fc17e9ffae609ee18c80a2b205ab15dec (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1-deactivate, org.label-schema.build-date=20240131, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.2, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=) 2024-02-07T12:58:38.684 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:58:38 smithi019 podman[134686]: 2024-02-07 12:58:38.682347475 +0000 UTC m=+0.438333513 container attach 200c4768291ea66ea8aae6c91079ca6fc17e9ffae609ee18c80a2b205ab15dec (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-02-07T12:58:39.094 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:58:38 smithi019 podman[134686]: 2024-02-07 12:58:38.882196657 +0000 UTC m=+0.638182698 container died 200c4768291ea66ea8aae6c91079ca6fc17e9ffae609ee18c80a2b205ab15dec (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240131, ceph=True) 2024-02-07T12:58:39.467 INFO:journalctl@ceph.osd.1.smithi019.stdout:Feb 07 12:58:39 smithi019 podman[134712]: 2024-02-07 12:58:39.435645201 +0000 UTC m=+0.548787181 container remove 200c4768291ea66ea8aae6c91079ca6fc17e9ffae609ee18c80a2b205ab15dec (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-1-deactivate, io.buildah.version=1.33.2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-02-07T12:58:39.472 DEBUG:teuthology.orchestra.run.smithi019:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.1.service' 2024-02-07T12:58:39.512 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-07T12:58:39.513 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-02-07T12:58:39.513 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-02-07T12:58:39.513 DEBUG:teuthology.orchestra.run.smithi019:> sudo systemctl stop ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.2 2024-02-07T12:58:39.840 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:58:39 smithi019 systemd[1]: Stopping Ceph osd.2 for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:58:40.094 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:58:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2[108124]: 2024-02-07T12:58:39.837+0000 7f1547728700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:58:40.094 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:58:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2[108124]: 2024-02-07T12:58:39.837+0000 7f1547728700 -1 osd.2 158 *** Got signal Terminated *** 2024-02-07T12:58:40.094 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:58:39 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2[108124]: 2024-02-07T12:58:39.837+0000 7f1547728700 -1 osd.2 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:58:43.844 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:58:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:58:43.553Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi172.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.172:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi019.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.19:8443: connect: connection refused" 2024-02-07T12:58:43.844 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:58:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:58:43.554Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi019.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.19:8443: connect: connection refused" 2024-02-07T12:58:43.844 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:58:43 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:58:43.554Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi172.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.172:8443: connect: connection refused" 2024-02-07T12:58:45.473 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:58:45 smithi019 podman[134790]: 2024-02-07 12:58:45.207668198 +0000 UTC m=+5.474828685 container died 90352897da859ee60b2030e764b1a4b756eeaa800a6daf3a66b77a9f6a5780b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240131, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, io.buildah.version=1.33.2) 2024-02-07T12:58:45.832 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:58:45 smithi019 podman[134790]: 2024-02-07 12:58:45.554769158 +0000 UTC m=+5.821929655 container cleanup 90352897da859ee60b2030e764b1a4b756eeaa800a6daf3a66b77a9f6a5780b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, ceph=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-02-07T12:58:45.832 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:58:45 smithi019 bash[134790]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2 2024-02-07T12:58:46.094 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:58:45 smithi019 podman[134807]: 2024-02-07 12:58:45.831063813 +0000 UTC m=+0.626403739 container remove 90352897da859ee60b2030e764b1a4b756eeaa800a6daf3a66b77a9f6a5780b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2, GIT_CLEAN=True, org.label-schema.build-date=20240131, io.buildah.version=1.33.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:58:46.435 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:58:46 smithi019 podman[134897]: 2024-02-07 12:58:46.039662476 +0000 UTC m=+0.017733885 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:58:46.435 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:58:46 smithi019 podman[134897]: 2024-02-07 12:58:46.156949625 +0000 UTC m=+0.135021025 container create fc5086cfa63ac566b6084500ae1d690525a2b1bee89645e14bde0da17ee03d44 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.2, ceph=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-02-07T12:58:46.435 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:58:46 smithi019 podman[134897]: 2024-02-07 12:58:46.351169781 +0000 UTC m=+0.329241191 container init fc5086cfa63ac566b6084500ae1d690525a2b1bee89645e14bde0da17ee03d44 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2-deactivate, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:58:46.436 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:58:46 smithi019 podman[134897]: 2024-02-07 12:58:46.354502597 +0000 UTC m=+0.332573999 container start fc5086cfa63ac566b6084500ae1d690525a2b1bee89645e14bde0da17ee03d44 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131, GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:58:46.436 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:58:46 smithi019 podman[134897]: 2024-02-07 12:58:46.434063619 +0000 UTC m=+0.412135034 container attach fc5086cfa63ac566b6084500ae1d690525a2b1bee89645e14bde0da17ee03d44 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2-deactivate, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240131, GIT_CLEAN=True) 2024-02-07T12:58:46.844 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:58:46 smithi019 podman[134897]: 2024-02-07 12:58:46.647999182 +0000 UTC m=+0.626070584 container died fc5086cfa63ac566b6084500ae1d690525a2b1bee89645e14bde0da17ee03d44 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2-deactivate, org.label-schema.build-date=20240131, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, CEPH_POINT_RELEASE=, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:58:47.196 DEBUG:teuthology.orchestra.run.smithi019:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.2.service' 2024-02-07T12:58:47.221 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:58:47 smithi019 podman[134923]: 2024-02-07 12:58:47.17926462 +0000 UTC m=+0.526461202 container remove fc5086cfa63ac566b6084500ae1d690525a2b1bee89645e14bde0da17ee03d44 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-2-deactivate, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-02-07T12:58:47.222 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:58:47 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.2.service: Deactivated successfully. 2024-02-07T12:58:47.222 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:58:47 smithi019 systemd[1]: Stopped Ceph osd.2 for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:58:47.222 INFO:journalctl@ceph.osd.2.smithi019.stdout:Feb 07 12:58:47 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.2.service: Consumed 5.215s CPU time. 2024-02-07T12:58:47.239 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-07T12:58:47.239 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-02-07T12:58:47.239 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-02-07T12:58:47.239 DEBUG:teuthology.orchestra.run.smithi019:> sudo systemctl stop ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.3 2024-02-07T12:58:47.549 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:58:47 smithi019 systemd[1]: Stopping Ceph osd.3 for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:58:47.844 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:58:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3[114945]: 2024-02-07T12:58:47.584+0000 7fda62c59700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:58:47.844 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:58:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3[114945]: 2024-02-07T12:58:47.584+0000 7fda62c59700 -1 osd.3 158 *** Got signal Terminated *** 2024-02-07T12:58:47.844 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:58:47 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3[114945]: 2024-02-07T12:58:47.584+0000 7fda62c59700 -1 osd.3 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:58:50.869 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:58:50 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:58:50.401+0000 7f26db647700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.356422+0000 front 2024-02-07T12:58:26.356491+0000 (oldest deadline 2024-02-07T12:58:49.856309+0000) 2024-02-07T12:58:51.618 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:58:51 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:58:51.445+0000 7f26db647700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.356422+0000 front 2024-02-07T12:58:26.356491+0000 (oldest deadline 2024-02-07T12:58:49.856309+0000) 2024-02-07T12:58:52.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:58:51 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:58:51.862+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.972278+0000 front 2024-02-07T12:58:26.972214+0000 (oldest deadline 2024-02-07T12:58:51.072116+0000) 2024-02-07T12:58:52.869 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:58:52 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:58:52.472+0000 7f26db647700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.356422+0000 front 2024-02-07T12:58:26.356491+0000 (oldest deadline 2024-02-07T12:58:49.856309+0000) 2024-02-07T12:58:53.209 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:58:52 smithi019 podman[135001]: 2024-02-07 12:58:52.94472097 +0000 UTC m=+5.477997558 container died 067cd6b3fd2f16613dfbb8ca6d758f04a49e7f52d2fc535c2a247db069de3bf9 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-02-07T12:58:53.369 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:58:52 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4[77818]: 2024-02-07T12:58:52.905+0000 7f53f8e44700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.469765+0000 front 2024-02-07T12:58:26.469643+0000 (oldest deadline 2024-02-07T12:58:52.369528+0000) 2024-02-07T12:58:53.369 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:58:52 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:58:52.883+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.972278+0000 front 2024-02-07T12:58:26.972214+0000 (oldest deadline 2024-02-07T12:58:51.072116+0000) 2024-02-07T12:58:53.555 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:58:53 smithi019 podman[135001]: 2024-02-07 12:58:53.306349832 +0000 UTC m=+5.839626421 container cleanup 067cd6b3fd2f16613dfbb8ca6d758f04a49e7f52d2fc535c2a247db069de3bf9 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, org.label-schema.build-date=20240131) 2024-02-07T12:58:53.555 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:58:53 smithi019 bash[135001]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3 2024-02-07T12:58:53.844 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:58:53 smithi019 podman[135018]: 2024-02-07 12:58:53.582698449 +0000 UTC m=+0.637239635 container remove 067cd6b3fd2f16613dfbb8ca6d758f04a49e7f52d2fc535c2a247db069de3bf9 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_CLEAN=True, io.buildah.version=1.33.2, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:58:53.844 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:58:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:58:53.553Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi019.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.19:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi172.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.172:8443: connect: connection refused" 2024-02-07T12:58:53.844 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:58:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:58:53.554Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi019.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.19:8443: connect: connection refused" 2024-02-07T12:58:53.844 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:58:53 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:58:53.554Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi172.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.172:8443: connect: connection refused" 2024-02-07T12:58:53.845 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:58:53 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:58:53.516+0000 7f26db647700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.356422+0000 front 2024-02-07T12:58:26.356491+0000 (oldest deadline 2024-02-07T12:58:49.856309+0000) 2024-02-07T12:58:54.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:58:53 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:58:53.842+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.972278+0000 front 2024-02-07T12:58:26.972214+0000 (oldest deadline 2024-02-07T12:58:51.072116+0000) 2024-02-07T12:58:54.119 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:58:53 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4[77818]: 2024-02-07T12:58:53.915+0000 7f53f8e44700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.469765+0000 front 2024-02-07T12:58:26.469643+0000 (oldest deadline 2024-02-07T12:58:52.369528+0000) 2024-02-07T12:58:54.178 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:58:53 smithi019 podman[135113]: 2024-02-07 12:58:53.828057111 +0000 UTC m=+0.018905055 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:58:54.178 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:58:53 smithi019 podman[135113]: 2024-02-07 12:58:53.950106101 +0000 UTC m=+0.140954093 container create cb21aaab98450762186e5745dd83f20af1cede4972b8a33640c1a663cda1f62d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3-deactivate, ceph=True, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , io.buildah.version=1.33.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:58:54.178 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:58:54 smithi019 podman[135113]: 2024-02-07 12:58:54.177299231 +0000 UTC m=+0.368147176 container init cb21aaab98450762186e5745dd83f20af1cede4972b8a33640c1a663cda1f62d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:58:54.492 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:58:54 smithi019 podman[135113]: 2024-02-07 12:58:54.180330572 +0000 UTC m=+0.371178510 container start cb21aaab98450762186e5745dd83f20af1cede4972b8a33640c1a663cda1f62d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3-deactivate, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:58:54.492 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:58:54 smithi019 podman[135113]: 2024-02-07 12:58:54.269178961 +0000 UTC m=+0.460026914 container attach cb21aaab98450762186e5745dd83f20af1cede4972b8a33640c1a663cda1f62d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:58:54.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:58:54 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:58:54.323+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:58:54.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:58:54 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:58:54.471+0000 7f26db647700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.356422+0000 front 2024-02-07T12:58:26.356491+0000 (oldest deadline 2024-02-07T12:58:49.856309+0000) 2024-02-07T12:58:54.781 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:58:54 smithi019 podman[135113]: 2024-02-07 12:58:54.491029913 +0000 UTC m=+0.681877855 container died cb21aaab98450762186e5745dd83f20af1cede4972b8a33640c1a663cda1f62d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , io.buildah.version=1.33.2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:58:55.033 INFO:journalctl@ceph.osd.3.smithi019.stdout:Feb 07 12:58:55 smithi019 podman[135139]: 2024-02-07 12:58:55.013884668 +0000 UTC m=+0.517274274 container remove cb21aaab98450762186e5745dd83f20af1cede4972b8a33640c1a663cda1f62d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.build-date=20240131, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, org.label-schema.vendor=CentOS, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-02-07T12:58:55.039 DEBUG:teuthology.orchestra.run.smithi019:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.3.service' 2024-02-07T12:58:55.081 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-07T12:58:55.081 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-02-07T12:58:55.081 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-02-07T12:58:55.081 DEBUG:teuthology.orchestra.run.smithi172:> sudo systemctl stop ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.4 2024-02-07T12:58:55.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:58:54 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:58:54.865+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.972278+0000 front 2024-02-07T12:58:26.972214+0000 (oldest deadline 2024-02-07T12:58:51.072116+0000) 2024-02-07T12:58:55.119 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:58:54 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4[77818]: 2024-02-07T12:58:54.866+0000 7f53f8e44700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.469765+0000 front 2024-02-07T12:58:26.469643+0000 (oldest deadline 2024-02-07T12:58:52.369528+0000) 2024-02-07T12:58:55.390 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:58:55 smithi172 systemd[1]: Stopping Ceph osd.4 for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:58:55.390 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:58:55 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:58:55.285+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:58:55.861 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:58:55 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:58:55.469+0000 7f26db647700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.356422+0000 front 2024-02-07T12:58:26.356491+0000 (oldest deadline 2024-02-07T12:58:49.856309+0000) 2024-02-07T12:58:55.861 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:58:55 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4[77818]: 2024-02-07T12:58:55.388+0000 7f53fc849700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:58:55.862 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:58:55 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4[77818]: 2024-02-07T12:58:55.388+0000 7f53fc849700 -1 osd.4 158 *** Got signal Terminated *** 2024-02-07T12:58:55.862 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:58:55 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4[77818]: 2024-02-07T12:58:55.388+0000 7f53fc849700 -1 osd.4 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:58:56.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:58:55 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:58:55.859+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.972278+0000 front 2024-02-07T12:58:26.972214+0000 (oldest deadline 2024-02-07T12:58:51.072116+0000) 2024-02-07T12:58:56.119 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:58:55 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4[77818]: 2024-02-07T12:58:55.896+0000 7f53f8e44700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.469765+0000 front 2024-02-07T12:58:26.469643+0000 (oldest deadline 2024-02-07T12:58:52.369528+0000) 2024-02-07T12:58:56.505 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:58:56 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:58:56.239+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:58:56.864 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:58:56 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:58:56.504+0000 7f26db647700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.356422+0000 front 2024-02-07T12:58:26.356491+0000 (oldest deadline 2024-02-07T12:58:49.856309+0000) 2024-02-07T12:58:57.119 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:58:56 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4[77818]: 2024-02-07T12:58:56.869+0000 7f53f8e44700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.469765+0000 front 2024-02-07T12:58:26.469643+0000 (oldest deadline 2024-02-07T12:58:52.369528+0000) 2024-02-07T12:58:57.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:58:56 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:58:56.862+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.972278+0000 front 2024-02-07T12:58:26.972214+0000 (oldest deadline 2024-02-07T12:58:51.072116+0000) 2024-02-07T12:58:57.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:58:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:58:57.483+0000 7f26db647700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.356422+0000 front 2024-02-07T12:58:26.356491+0000 (oldest deadline 2024-02-07T12:58:49.856309+0000) 2024-02-07T12:58:57.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:58:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:58:57.248+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:58:58.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:58:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:58:57.816+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.972278+0000 front 2024-02-07T12:58:26.972214+0000 (oldest deadline 2024-02-07T12:58:51.072116+0000) 2024-02-07T12:58:58.119 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:58:57 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4[77818]: 2024-02-07T12:58:57.850+0000 7f53f8e44700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.469765+0000 front 2024-02-07T12:58:26.469643+0000 (oldest deadline 2024-02-07T12:58:52.369528+0000) 2024-02-07T12:58:58.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:58:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:58:58.220+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:58:58.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:58:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:58:58.443+0000 7f26db647700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.356422+0000 front 2024-02-07T12:58:26.356491+0000 (oldest deadline 2024-02-07T12:58:49.856309+0000) 2024-02-07T12:58:59.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:58:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:58:58.789+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.972278+0000 front 2024-02-07T12:58:26.972214+0000 (oldest deadline 2024-02-07T12:58:51.072116+0000) 2024-02-07T12:58:59.119 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:58:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4[77818]: 2024-02-07T12:58:58.859+0000 7f53f8e44700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.469765+0000 front 2024-02-07T12:58:26.469643+0000 (oldest deadline 2024-02-07T12:58:52.369528+0000) 2024-02-07T12:58:59.119 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:58:58 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4[77818]: 2024-02-07T12:58:58.859+0000 7f53f8e44700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:32.369963+0000 front 2024-02-07T12:58:32.369984+0000 (oldest deadline 2024-02-07T12:58:58.269903+0000) 2024-02-07T12:58:59.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:58:59 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:58:59.475+0000 7f26db647700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.356422+0000 front 2024-02-07T12:58:26.356491+0000 (oldest deadline 2024-02-07T12:58:49.856309+0000) 2024-02-07T12:58:59.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:58:59 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:58:59.242+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:58:59.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:58:59 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:58:59.242+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:34.015766+0000 front 2024-02-07T12:58:34.015850+0000 (oldest deadline 2024-02-07T12:58:58.616037+0000) 2024-02-07T12:59:00.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:58:59 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:58:59.816+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.972278+0000 front 2024-02-07T12:58:26.972214+0000 (oldest deadline 2024-02-07T12:58:51.072116+0000) 2024-02-07T12:59:00.119 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:58:59 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4[77818]: 2024-02-07T12:58:59.816+0000 7f53f8e44700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.469765+0000 front 2024-02-07T12:58:26.469643+0000 (oldest deadline 2024-02-07T12:58:52.369528+0000) 2024-02-07T12:59:00.119 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:58:59 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4[77818]: 2024-02-07T12:58:59.816+0000 7f53f8e44700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:32.369963+0000 front 2024-02-07T12:58:32.369984+0000 (oldest deadline 2024-02-07T12:58:58.269903+0000) 2024-02-07T12:59:00.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:59:00.464+0000 7f26db647700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.356422+0000 front 2024-02-07T12:58:26.356491+0000 (oldest deadline 2024-02-07T12:58:49.856309+0000) 2024-02-07T12:59:00.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:00.288+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:59:00.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:00.288+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:34.015766+0000 front 2024-02-07T12:58:34.015850+0000 (oldest deadline 2024-02-07T12:58:58.616037+0000) 2024-02-07T12:59:01.119 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:59:00 smithi172 podman[100686]: 2024-02-07 12:59:00.856390032 +0000 UTC m=+5.577243336 container died dbb47a8ad5ac9544a15d786f62a16c1bccaa461086e55aa04ea36b297cecf43d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:59:01.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:00.830+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.972278+0000 front 2024-02-07T12:58:26.972214+0000 (oldest deadline 2024-02-07T12:58:51.072116+0000) 2024-02-07T12:59:01.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:00 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:00.830+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:31.072761+0000 front 2024-02-07T12:58:31.072701+0000 (oldest deadline 2024-02-07T12:58:59.873154+0000) 2024-02-07T12:59:01.481 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:01.332+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:59:01.481 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:01.332+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:34.015766+0000 front 2024-02-07T12:58:34.015850+0000 (oldest deadline 2024-02-07T12:58:58.616037+0000) 2024-02-07T12:59:01.481 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:59:01 smithi172 podman[100686]: 2024-02-07 12:59:01.271196108 +0000 UTC m=+5.992049450 container cleanup dbb47a8ad5ac9544a15d786f62a16c1bccaa461086e55aa04ea36b297cecf43d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2) 2024-02-07T12:59:01.481 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:59:01 smithi172 bash[100686]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4 2024-02-07T12:59:01.800 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:59:01.479+0000 7f26db647700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.356422+0000 front 2024-02-07T12:58:26.356491+0000 (oldest deadline 2024-02-07T12:58:49.856309+0000) 2024-02-07T12:59:01.801 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:59:01 smithi172 podman[100701]: 2024-02-07 12:59:01.502338531 +0000 UTC m=+0.642683093 container remove dbb47a8ad5ac9544a15d786f62a16c1bccaa461086e55aa04ea36b297cecf43d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4, ceph=True, io.buildah.version=1.33.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:59:02.077 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:01.799+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.972278+0000 front 2024-02-07T12:58:26.972214+0000 (oldest deadline 2024-02-07T12:58:51.072116+0000) 2024-02-07T12:59:02.077 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:01 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:01.799+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:31.072761+0000 front 2024-02-07T12:58:31.072701+0000 (oldest deadline 2024-02-07T12:58:59.873154+0000) 2024-02-07T12:59:02.077 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:59:01 smithi172 podman[100794]: 2024-02-07 12:59:01.837112528 +0000 UTC m=+0.017870511 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:59:02.077 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:59:01 smithi172 podman[100794]: 2024-02-07 12:59:01.956913145 +0000 UTC m=+0.137671113 container create 5de7aa124bbd033057fc24510ac5fe2a1b1cf9e713b546ad351792fdc1d40807 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-02-07T12:59:02.353 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:02 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:02.306+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:59:02.353 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:02 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:02.306+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:34.015766+0000 front 2024-02-07T12:58:34.015850+0000 (oldest deadline 2024-02-07T12:58:58.616037+0000) 2024-02-07T12:59:02.353 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:59:02 smithi172 podman[100794]: 2024-02-07 12:59:02.268789956 +0000 UTC m=+0.449547937 container init 5de7aa124bbd033057fc24510ac5fe2a1b1cf9e713b546ad351792fdc1d40807 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4-deactivate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2) 2024-02-07T12:59:02.353 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:59:02 smithi172 podman[100794]: 2024-02-07 12:59:02.272146612 +0000 UTC m=+0.452904582 container start 5de7aa124bbd033057fc24510ac5fe2a1b1cf9e713b546ad351792fdc1d40807 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4-deactivate, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131) 2024-02-07T12:59:02.354 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:59:02 smithi172 podman[100794]: 2024-02-07 12:59:02.351729109 +0000 UTC m=+0.532487078 container attach 5de7aa124bbd033057fc24510ac5fe2a1b1cf9e713b546ad351792fdc1d40807 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4-deactivate, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-02-07T12:59:02.619 INFO:journalctl@ceph.osd.4.smithi172.stdout:Feb 07 12:59:02 smithi172 podman[100794]: 2024-02-07 12:59:02.563848294 +0000 UTC m=+0.744606267 container died 5de7aa124bbd033057fc24510ac5fe2a1b1cf9e713b546ad351792fdc1d40807 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-4-deactivate, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.2) 2024-02-07T12:59:02.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:02 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:59:02.508+0000 7f26db647700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.356422+0000 front 2024-02-07T12:58:26.356491+0000 (oldest deadline 2024-02-07T12:58:49.856309+0000) 2024-02-07T12:59:02.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:02 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:59:02.508+0000 7f26db647700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:35.757588+0000 front 2024-02-07T12:58:35.757603+0000 (oldest deadline 2024-02-07T12:59:01.657500+0000) 2024-02-07T12:59:03.081 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:02 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:02.817+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.972278+0000 front 2024-02-07T12:58:26.972214+0000 (oldest deadline 2024-02-07T12:58:51.072116+0000) 2024-02-07T12:59:03.081 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:02 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:02.817+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:31.072761+0000 front 2024-02-07T12:58:31.072701+0000 (oldest deadline 2024-02-07T12:58:59.873154+0000) 2024-02-07T12:59:03.101 DEBUG:teuthology.orchestra.run.smithi172:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.4.service' 2024-02-07T12:59:03.183 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-07T12:59:03.183 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-02-07T12:59:03.183 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-02-07T12:59:03.183 DEBUG:teuthology.orchestra.run.smithi172:> sudo systemctl stop ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.5 2024-02-07T12:59:03.344 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:03 smithi172 systemd[1]: Stopping Ceph osd.5 for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:59:03.344 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:03 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:03.262+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:59:03.345 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:03 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:03.262+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:34.015766+0000 front 2024-02-07T12:58:34.015850+0000 (oldest deadline 2024-02-07T12:58:58.616037+0000) 2024-02-07T12:59:03.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:03 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:59:03.496+0000 7f26db647700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.356422+0000 front 2024-02-07T12:58:26.356491+0000 (oldest deadline 2024-02-07T12:58:49.856309+0000) 2024-02-07T12:59:03.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:03 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:59:03.496+0000 7f26db647700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:35.757588+0000 front 2024-02-07T12:58:35.757603+0000 (oldest deadline 2024-02-07T12:59:01.657500+0000) 2024-02-07T12:59:03.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:03 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:59:03.499+0000 7f26df04c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:59:03.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:03 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:59:03.499+0000 7f26df04c700 -1 osd.5 158 *** Got signal Terminated *** 2024-02-07T12:59:03.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:03 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:59:03.499+0000 7f26df04c700 -1 osd.5 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:59:03.844 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:59:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:59:03.553Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=5 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi019.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.19:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi172.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.172:8443: connect: connection refused" 2024-02-07T12:59:03.844 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:59:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:59:03.554Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi019.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.19:8443: connect: connection refused" 2024-02-07T12:59:03.844 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:59:03 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:59:03.555Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi172.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.172:8443: connect: connection refused" 2024-02-07T12:59:04.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:03 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:03.856+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.972278+0000 front 2024-02-07T12:58:26.972214+0000 (oldest deadline 2024-02-07T12:58:51.072116+0000) 2024-02-07T12:59:04.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:03 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:03.856+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:31.072761+0000 front 2024-02-07T12:58:31.072701+0000 (oldest deadline 2024-02-07T12:58:59.873154+0000) 2024-02-07T12:59:04.518 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:04.302+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:59:04.518 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:04.302+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:34.015766+0000 front 2024-02-07T12:58:34.015850+0000 (oldest deadline 2024-02-07T12:58:58.616037+0000) 2024-02-07T12:59:04.836 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:59:04.516+0000 7f26db647700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.356422+0000 front 2024-02-07T12:58:26.356491+0000 (oldest deadline 2024-02-07T12:58:49.856309+0000) 2024-02-07T12:59:04.836 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:59:04.516+0000 7f26db647700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:35.757588+0000 front 2024-02-07T12:58:35.757603+0000 (oldest deadline 2024-02-07T12:59:01.657500+0000) 2024-02-07T12:59:05.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:04.834+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.972278+0000 front 2024-02-07T12:58:26.972214+0000 (oldest deadline 2024-02-07T12:58:51.072116+0000) 2024-02-07T12:59:05.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:04 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:04.834+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:31.072761+0000 front 2024-02-07T12:58:31.072701+0000 (oldest deadline 2024-02-07T12:58:59.873154+0000) 2024-02-07T12:59:05.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:05 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:05.285+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:59:05.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:05 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:05.285+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:34.015766+0000 front 2024-02-07T12:58:34.015850+0000 (oldest deadline 2024-02-07T12:58:58.616037+0000) 2024-02-07T12:59:05.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:05 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:59:05.501+0000 7f26db647700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.356422+0000 front 2024-02-07T12:58:26.356491+0000 (oldest deadline 2024-02-07T12:58:49.856309+0000) 2024-02-07T12:59:05.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:05 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:59:05.501+0000 7f26db647700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:35.757588+0000 front 2024-02-07T12:58:35.757603+0000 (oldest deadline 2024-02-07T12:59:01.657500+0000) 2024-02-07T12:59:06.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:05 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:05.852+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.972278+0000 front 2024-02-07T12:58:26.972214+0000 (oldest deadline 2024-02-07T12:58:51.072116+0000) 2024-02-07T12:59:06.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:05 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:05.852+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:31.072761+0000 front 2024-02-07T12:58:31.072701+0000 (oldest deadline 2024-02-07T12:58:59.873154+0000) 2024-02-07T12:59:06.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:06 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:06.289+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:59:06.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:06 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:06.289+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:34.015766+0000 front 2024-02-07T12:58:34.015850+0000 (oldest deadline 2024-02-07T12:58:58.616037+0000) 2024-02-07T12:59:06.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:06 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:59:06.457+0000 7f26db647700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.356422+0000 front 2024-02-07T12:58:26.356491+0000 (oldest deadline 2024-02-07T12:58:49.856309+0000) 2024-02-07T12:59:06.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:06 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:59:06.457+0000 7f26db647700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:35.757588+0000 front 2024-02-07T12:58:35.757603+0000 (oldest deadline 2024-02-07T12:59:01.657500+0000) 2024-02-07T12:59:07.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:06 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:06.817+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.972278+0000 front 2024-02-07T12:58:26.972214+0000 (oldest deadline 2024-02-07T12:58:51.072116+0000) 2024-02-07T12:59:07.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:06 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:06.817+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:31.072761+0000 front 2024-02-07T12:58:31.072701+0000 (oldest deadline 2024-02-07T12:58:59.873154+0000) 2024-02-07T12:59:07.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:07.265+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:59:07.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:07.265+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:34.015766+0000 front 2024-02-07T12:58:34.015850+0000 (oldest deadline 2024-02-07T12:58:58.616037+0000) 2024-02-07T12:59:07.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:07.265+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:43.117882+0000 front 2024-02-07T12:58:43.117961+0000 (oldest deadline 2024-02-07T12:59:07.217760+0000) 2024-02-07T12:59:07.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:59:07.419+0000 7f26db647700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.356422+0000 front 2024-02-07T12:58:26.356491+0000 (oldest deadline 2024-02-07T12:58:49.856309+0000) 2024-02-07T12:59:07.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:59:07.419+0000 7f26db647700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:35.757588+0000 front 2024-02-07T12:58:35.757603+0000 (oldest deadline 2024-02-07T12:59:01.657500+0000) 2024-02-07T12:59:08.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:07.805+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.972278+0000 front 2024-02-07T12:58:26.972214+0000 (oldest deadline 2024-02-07T12:58:51.072116+0000) 2024-02-07T12:59:08.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:07.805+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:31.072761+0000 front 2024-02-07T12:58:31.072701+0000 (oldest deadline 2024-02-07T12:58:59.873154+0000) 2024-02-07T12:59:08.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:07 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:07.805+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:44.574083+0000 front 2024-02-07T12:58:44.574063+0000 (oldest deadline 2024-02-07T12:59:07.473951+0000) 2024-02-07T12:59:08.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:08 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:08.271+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:59:08.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:08 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:08.271+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:34.015766+0000 front 2024-02-07T12:58:34.015850+0000 (oldest deadline 2024-02-07T12:58:58.616037+0000) 2024-02-07T12:59:08.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:08 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:08.271+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:43.117882+0000 front 2024-02-07T12:58:43.117961+0000 (oldest deadline 2024-02-07T12:59:07.217760+0000) 2024-02-07T12:59:08.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:08 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:59:08.387+0000 7f26db647700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.356422+0000 front 2024-02-07T12:58:26.356491+0000 (oldest deadline 2024-02-07T12:58:49.856309+0000) 2024-02-07T12:59:08.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:08 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:59:08.387+0000 7f26db647700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:35.757588+0000 front 2024-02-07T12:58:35.757603+0000 (oldest deadline 2024-02-07T12:59:01.657500+0000) 2024-02-07T12:59:08.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:08 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5[82362]: 2024-02-07T12:59:08.387+0000 7f26db647700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:41.657946+0000 front 2024-02-07T12:58:41.657986+0000 (oldest deadline 2024-02-07T12:59:07.557874+0000) 2024-02-07T12:59:09.119 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:09 smithi172 podman[100900]: 2024-02-07 12:59:09.014276538 +0000 UTC m=+5.603049408 container died d139d35fec5787bb892e1f7524804b305300c589c4af4378cb4ed77ff809e13c (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, org.label-schema.schema-version=1.0) 2024-02-07T12:59:09.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:08 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:08.833+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.972278+0000 front 2024-02-07T12:58:26.972214+0000 (oldest deadline 2024-02-07T12:58:51.072116+0000) 2024-02-07T12:59:09.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:08 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:08.833+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:31.072761+0000 front 2024-02-07T12:58:31.072701+0000 (oldest deadline 2024-02-07T12:58:59.873154+0000) 2024-02-07T12:59:09.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:08 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:08.833+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:44.574083+0000 front 2024-02-07T12:58:44.574063+0000 (oldest deadline 2024-02-07T12:59:07.473951+0000) 2024-02-07T12:59:09.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:09 smithi172 podman[100900]: 2024-02-07 12:59:09.449617764 +0000 UTC m=+6.038390681 container cleanup d139d35fec5787bb892e1f7524804b305300c589c4af4378cb4ed77ff809e13c (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, maintainer=Guillaume Abrioux ) 2024-02-07T12:59:09.619 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:09 smithi172 bash[100900]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5 2024-02-07T12:59:09.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:09 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:09.244+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:59:09.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:09 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:09.244+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:34.015766+0000 front 2024-02-07T12:58:34.015850+0000 (oldest deadline 2024-02-07T12:58:58.616037+0000) 2024-02-07T12:59:09.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:09 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:09.244+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:43.117882+0000 front 2024-02-07T12:58:43.117961+0000 (oldest deadline 2024-02-07T12:59:07.217760+0000) 2024-02-07T12:59:10.119 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:09 smithi172 podman[100928]: 2024-02-07 12:59:09.759837012 +0000 UTC m=+0.738588336 container remove d139d35fec5787bb892e1f7524804b305300c589c4af4378cb4ed77ff809e13c (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:59:10.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:09 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:09.839+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.972278+0000 front 2024-02-07T12:58:26.972214+0000 (oldest deadline 2024-02-07T12:58:51.072116+0000) 2024-02-07T12:59:10.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:09 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:09.839+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:31.072761+0000 front 2024-02-07T12:58:31.072701+0000 (oldest deadline 2024-02-07T12:58:59.873154+0000) 2024-02-07T12:59:10.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:09 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:09.839+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:44.574083+0000 front 2024-02-07T12:58:44.574063+0000 (oldest deadline 2024-02-07T12:59:07.473951+0000) 2024-02-07T12:59:10.497 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:10 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:10.259+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:59:10.497 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:10 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:10.259+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:34.015766+0000 front 2024-02-07T12:58:34.015850+0000 (oldest deadline 2024-02-07T12:58:58.616037+0000) 2024-02-07T12:59:10.497 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:10 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:10.259+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:43.117882+0000 front 2024-02-07T12:58:43.117961+0000 (oldest deadline 2024-02-07T12:59:07.217760+0000) 2024-02-07T12:59:10.498 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:10 smithi172 podman[101019]: 2024-02-07 12:59:10.218850007 +0000 UTC m=+0.112100153 container create 4b6e9f1cf544e7a42089f3b272682c2a803cbb84b9f5aa2fd215051f3a449a32 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True) 2024-02-07T12:59:10.498 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:10 smithi172 podman[101019]: 2024-02-07 12:59:10.124774815 +0000 UTC m=+0.018024928 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:59:10.498 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:10 smithi172 podman[101019]: 2024-02-07 12:59:10.40499057 +0000 UTC m=+0.298240691 container init 4b6e9f1cf544e7a42089f3b272682c2a803cbb84b9f5aa2fd215051f3a449a32 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:59:10.498 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:10 smithi172 podman[101019]: 2024-02-07 12:59:10.408303828 +0000 UTC m=+0.301553933 container start 4b6e9f1cf544e7a42089f3b272682c2a803cbb84b9f5aa2fd215051f3a449a32 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, ceph=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:59:10.498 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:10 smithi172 podman[101019]: 2024-02-07 12:59:10.496110203 +0000 UTC m=+0.389360331 container attach 4b6e9f1cf544e7a42089f3b272682c2a803cbb84b9f5aa2fd215051f3a449a32 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, io.buildah.version=1.33.2, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:59:10.835 INFO:journalctl@ceph.osd.5.smithi172.stdout:Feb 07 12:59:10 smithi172 podman[101019]: 2024-02-07 12:59:10.709374094 +0000 UTC m=+0.602624197 container died 4b6e9f1cf544e7a42089f3b272682c2a803cbb84b9f5aa2fd215051f3a449a32 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-5-deactivate, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-02-07T12:59:11.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:10 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:10.832+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.972278+0000 front 2024-02-07T12:58:26.972214+0000 (oldest deadline 2024-02-07T12:58:51.072116+0000) 2024-02-07T12:59:11.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:10 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:10.832+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:31.072761+0000 front 2024-02-07T12:58:31.072701+0000 (oldest deadline 2024-02-07T12:58:59.873154+0000) 2024-02-07T12:59:11.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:10 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:10.832+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:44.574083+0000 front 2024-02-07T12:58:44.574063+0000 (oldest deadline 2024-02-07T12:59:07.473951+0000) 2024-02-07T12:59:11.357 DEBUG:teuthology.orchestra.run.smithi172:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.5.service' 2024-02-07T12:59:11.398 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-07T12:59:11.398 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-02-07T12:59:11.398 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-02-07T12:59:11.398 DEBUG:teuthology.orchestra.run.smithi172:> sudo systemctl stop ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.6 2024-02-07T12:59:11.478 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:11 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:11.214+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:59:11.479 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:11 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:11.214+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:34.015766+0000 front 2024-02-07T12:58:34.015850+0000 (oldest deadline 2024-02-07T12:58:58.616037+0000) 2024-02-07T12:59:11.479 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:11 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:11.214+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:43.117882+0000 front 2024-02-07T12:58:43.117961+0000 (oldest deadline 2024-02-07T12:59:07.217760+0000) 2024-02-07T12:59:11.802 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:11 smithi172 systemd[1]: Stopping Ceph osd.6 for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:59:11.802 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:11 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:11.710+0000 7f87de07d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:59:11.802 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:11 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:11.710+0000 7f87de07d700 -1 osd.6 158 *** Got signal Terminated *** 2024-02-07T12:59:11.802 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:11 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:11.710+0000 7f87de07d700 -1 osd.6 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:59:11.802 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:11 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:11.799+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.972278+0000 front 2024-02-07T12:58:26.972214+0000 (oldest deadline 2024-02-07T12:58:51.072116+0000) 2024-02-07T12:59:11.802 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:11 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:11.799+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:31.072761+0000 front 2024-02-07T12:58:31.072701+0000 (oldest deadline 2024-02-07T12:58:59.873154+0000) 2024-02-07T12:59:11.802 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:11 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:11.799+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:44.574083+0000 front 2024-02-07T12:58:44.574063+0000 (oldest deadline 2024-02-07T12:59:07.473951+0000) 2024-02-07T12:59:12.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:12 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:12.217+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:59:12.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:12 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:12.217+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:34.015766+0000 front 2024-02-07T12:58:34.015850+0000 (oldest deadline 2024-02-07T12:58:58.616037+0000) 2024-02-07T12:59:12.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:12 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:12.217+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:43.117882+0000 front 2024-02-07T12:58:43.117961+0000 (oldest deadline 2024-02-07T12:59:07.217760+0000) 2024-02-07T12:59:13.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:12 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:12.815+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.972278+0000 front 2024-02-07T12:58:26.972214+0000 (oldest deadline 2024-02-07T12:58:51.072116+0000) 2024-02-07T12:59:13.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:12 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:12.815+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:31.072761+0000 front 2024-02-07T12:58:31.072701+0000 (oldest deadline 2024-02-07T12:58:59.873154+0000) 2024-02-07T12:59:13.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:12 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:12.815+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:44.574083+0000 front 2024-02-07T12:58:44.574063+0000 (oldest deadline 2024-02-07T12:59:07.473951+0000) 2024-02-07T12:59:13.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:13 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:13.169+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:59:13.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:13 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:13.169+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:34.015766+0000 front 2024-02-07T12:58:34.015850+0000 (oldest deadline 2024-02-07T12:58:58.616037+0000) 2024-02-07T12:59:13.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:13 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:13.169+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:43.117882+0000 front 2024-02-07T12:58:43.117961+0000 (oldest deadline 2024-02-07T12:59:07.217760+0000) 2024-02-07T12:59:13.844 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:59:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:59:13.554Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=5 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi019.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.19:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi172.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.172:8443: connect: connection refused" 2024-02-07T12:59:13.844 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:59:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:59:13.555Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi019.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.19:8443: connect: connection refused" 2024-02-07T12:59:13.844 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:59:13 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:59:13.555Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi172.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.172:8443: connect: connection refused" 2024-02-07T12:59:14.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:13 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:13.816+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.972278+0000 front 2024-02-07T12:58:26.972214+0000 (oldest deadline 2024-02-07T12:58:51.072116+0000) 2024-02-07T12:59:14.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:13 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:13.816+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:31.072761+0000 front 2024-02-07T12:58:31.072701+0000 (oldest deadline 2024-02-07T12:58:59.873154+0000) 2024-02-07T12:59:14.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:13 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:13.816+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:44.574083+0000 front 2024-02-07T12:58:44.574063+0000 (oldest deadline 2024-02-07T12:59:07.473951+0000) 2024-02-07T12:59:14.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:14.130+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:59:14.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:14.130+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:34.015766+0000 front 2024-02-07T12:58:34.015850+0000 (oldest deadline 2024-02-07T12:58:58.616037+0000) 2024-02-07T12:59:14.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:14.130+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:43.117882+0000 front 2024-02-07T12:58:43.117961+0000 (oldest deadline 2024-02-07T12:59:07.217760+0000) 2024-02-07T12:59:14.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:14.130+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6830 osd.3 since back 2024-02-07T12:58:50.519664+0000 front 2024-02-07T12:58:50.519689+0000 (oldest deadline 2024-02-07T12:59:13.419542+0000) 2024-02-07T12:59:15.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:14.786+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.972278+0000 front 2024-02-07T12:58:26.972214+0000 (oldest deadline 2024-02-07T12:58:51.072116+0000) 2024-02-07T12:59:15.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:14.786+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:31.072761+0000 front 2024-02-07T12:58:31.072701+0000 (oldest deadline 2024-02-07T12:58:59.873154+0000) 2024-02-07T12:59:15.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:14.786+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:44.574083+0000 front 2024-02-07T12:58:44.574063+0000 (oldest deadline 2024-02-07T12:59:07.473951+0000) 2024-02-07T12:59:15.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:14 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:14.786+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6830 osd.3 since back 2024-02-07T12:58:48.575046+0000 front 2024-02-07T12:58:48.575008+0000 (oldest deadline 2024-02-07T12:59:13.874881+0000) 2024-02-07T12:59:15.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:15.146+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:59:15.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:15.146+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:34.015766+0000 front 2024-02-07T12:58:34.015850+0000 (oldest deadline 2024-02-07T12:58:58.616037+0000) 2024-02-07T12:59:15.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:15.146+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:43.117882+0000 front 2024-02-07T12:58:43.117961+0000 (oldest deadline 2024-02-07T12:59:07.217760+0000) 2024-02-07T12:59:15.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:15.146+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6830 osd.3 since back 2024-02-07T12:58:50.519664+0000 front 2024-02-07T12:58:50.519689+0000 (oldest deadline 2024-02-07T12:59:13.419542+0000) 2024-02-07T12:59:16.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:15.790+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:26.972278+0000 front 2024-02-07T12:58:26.972214+0000 (oldest deadline 2024-02-07T12:58:51.072116+0000) 2024-02-07T12:59:16.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:15.790+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:31.072761+0000 front 2024-02-07T12:58:31.072701+0000 (oldest deadline 2024-02-07T12:58:59.873154+0000) 2024-02-07T12:59:16.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:15.790+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:44.574083+0000 front 2024-02-07T12:58:44.574063+0000 (oldest deadline 2024-02-07T12:59:07.473951+0000) 2024-02-07T12:59:16.119 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:15 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6[86903]: 2024-02-07T12:59:15.790+0000 7f87d6670700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.19:6830 osd.3 since back 2024-02-07T12:58:48.575046+0000 front 2024-02-07T12:58:48.575008+0000 (oldest deadline 2024-02-07T12:59:13.874881+0000) 2024-02-07T12:59:16.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:16 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:16.134+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:59:16.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:16 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:16.134+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:34.015766+0000 front 2024-02-07T12:58:34.015850+0000 (oldest deadline 2024-02-07T12:58:58.616037+0000) 2024-02-07T12:59:16.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:16 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:16.134+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:43.117882+0000 front 2024-02-07T12:58:43.117961+0000 (oldest deadline 2024-02-07T12:59:07.217760+0000) 2024-02-07T12:59:16.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:16 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:16.134+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6830 osd.3 since back 2024-02-07T12:58:50.519664+0000 front 2024-02-07T12:58:50.519689+0000 (oldest deadline 2024-02-07T12:59:13.419542+0000) 2024-02-07T12:59:17.363 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:17 smithi172 podman[101124]: 2024-02-07 12:59:17.117896266 +0000 UTC m=+5.503219579 container died 219cf25240d51c78f8d79bc29de4964539d2e7f49d69d873b3bc7df0aedd8438 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True) 2024-02-07T12:59:17.363 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:17.103+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:59:17.363 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:17.103+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:34.015766+0000 front 2024-02-07T12:58:34.015850+0000 (oldest deadline 2024-02-07T12:58:58.616037+0000) 2024-02-07T12:59:17.363 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:17.103+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:43.117882+0000 front 2024-02-07T12:58:43.117961+0000 (oldest deadline 2024-02-07T12:59:07.217760+0000) 2024-02-07T12:59:17.364 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:17 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:17.103+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6830 osd.3 since back 2024-02-07T12:58:50.519664+0000 front 2024-02-07T12:58:50.519689+0000 (oldest deadline 2024-02-07T12:59:13.419542+0000) 2024-02-07T12:59:17.619 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:17 smithi172 podman[101124]: 2024-02-07 12:59:17.468841359 +0000 UTC m=+5.854164661 container cleanup 219cf25240d51c78f8d79bc29de4964539d2e7f49d69d873b3bc7df0aedd8438 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_CLEAN=True) 2024-02-07T12:59:17.619 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:17 smithi172 bash[101124]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6 2024-02-07T12:59:18.059 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:17 smithi172 podman[101143]: 2024-02-07 12:59:17.770376442 +0000 UTC m=+0.652658739 container remove 219cf25240d51c78f8d79bc29de4964539d2e7f49d69d873b3bc7df0aedd8438 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240131, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2) 2024-02-07T12:59:18.369 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:18 smithi172 podman[101237]: 2024-02-07 12:59:17.990032936 +0000 UTC m=+0.018428377 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:59:18.369 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:18 smithi172 podman[101237]: 2024-02-07 12:59:18.096019519 +0000 UTC m=+0.124414954 container create 4fc1de6196e90107cf86118e623808253986d87282a37fb6c42b6671f0c177d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.2) 2024-02-07T12:59:18.369 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:18 smithi172 podman[101237]: 2024-02-07 12:59:18.29828902 +0000 UTC m=+0.326684456 container init 4fc1de6196e90107cf86118e623808253986d87282a37fb6c42b6671f0c177d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-02-07T12:59:18.369 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:18 smithi172 podman[101237]: 2024-02-07 12:59:18.301386358 +0000 UTC m=+0.329781801 container start 4fc1de6196e90107cf86118e623808253986d87282a37fb6c42b6671f0c177d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6-deactivate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS) 2024-02-07T12:59:18.369 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:18 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:18.057+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:59:18.370 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:18 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:18.057+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:34.015766+0000 front 2024-02-07T12:58:34.015850+0000 (oldest deadline 2024-02-07T12:58:58.616037+0000) 2024-02-07T12:59:18.370 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:18 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:18.057+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:43.117882+0000 front 2024-02-07T12:58:43.117961+0000 (oldest deadline 2024-02-07T12:59:07.217760+0000) 2024-02-07T12:59:18.370 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:18 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:18.057+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6830 osd.3 since back 2024-02-07T12:58:50.519664+0000 front 2024-02-07T12:58:50.519689+0000 (oldest deadline 2024-02-07T12:59:13.419542+0000) 2024-02-07T12:59:18.869 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:18 smithi172 podman[101237]: 2024-02-07 12:59:18.391730822 +0000 UTC m=+0.420126269 container attach 4fc1de6196e90107cf86118e623808253986d87282a37fb6c42b6671f0c177d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6-deactivate, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131) 2024-02-07T12:59:18.869 INFO:journalctl@ceph.osd.6.smithi172.stdout:Feb 07 12:59:18 smithi172 podman[101237]: 2024-02-07 12:59:18.587387641 +0000 UTC m=+0.615783080 container died 4fc1de6196e90107cf86118e623808253986d87282a37fb6c42b6671f0c177d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-02-07T12:59:19.128 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:19 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:19.068+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:59:19.128 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:19 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:19.068+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:34.015766+0000 front 2024-02-07T12:58:34.015850+0000 (oldest deadline 2024-02-07T12:58:58.616037+0000) 2024-02-07T12:59:19.128 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:19 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:19.068+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:43.117882+0000 front 2024-02-07T12:58:43.117961+0000 (oldest deadline 2024-02-07T12:59:07.217760+0000) 2024-02-07T12:59:19.128 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:19 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:19.068+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6830 osd.3 since back 2024-02-07T12:58:50.519664+0000 front 2024-02-07T12:58:50.519689+0000 (oldest deadline 2024-02-07T12:59:13.419542+0000) 2024-02-07T12:59:19.154 DEBUG:teuthology.orchestra.run.smithi172:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.6.service' 2024-02-07T12:59:19.235 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-07T12:59:19.235 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-02-07T12:59:19.235 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-02-07T12:59:19.236 DEBUG:teuthology.orchestra.run.smithi172:> sudo systemctl stop ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.7 2024-02-07T12:59:19.548 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:19 smithi172 systemd[1]: Stopping Ceph osd.7 for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:59:19.548 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:19 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:19.545+0000 7f1bb0ee9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:59:19.869 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:19 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:19.545+0000 7f1bb0ee9700 -1 osd.7 158 *** Got signal Terminated *** 2024-02-07T12:59:19.869 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:19 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:19.545+0000 7f1bb0ee9700 -1 osd.7 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:59:20.369 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:20 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:20.041+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:59:20.369 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:20 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:20.041+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:34.015766+0000 front 2024-02-07T12:58:34.015850+0000 (oldest deadline 2024-02-07T12:58:58.616037+0000) 2024-02-07T12:59:20.369 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:20 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:20.041+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:43.117882+0000 front 2024-02-07T12:58:43.117961+0000 (oldest deadline 2024-02-07T12:59:07.217760+0000) 2024-02-07T12:59:20.369 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:20 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:20.041+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6830 osd.3 since back 2024-02-07T12:58:50.519664+0000 front 2024-02-07T12:58:50.519689+0000 (oldest deadline 2024-02-07T12:59:13.419542+0000) 2024-02-07T12:59:21.369 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:21 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:21.063+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:59:21.369 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:21 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:21.063+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:34.015766+0000 front 2024-02-07T12:58:34.015850+0000 (oldest deadline 2024-02-07T12:58:58.616037+0000) 2024-02-07T12:59:21.369 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:21 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:21.063+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:43.117882+0000 front 2024-02-07T12:58:43.117961+0000 (oldest deadline 2024-02-07T12:59:07.217760+0000) 2024-02-07T12:59:21.369 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:21 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:21.063+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6830 osd.3 since back 2024-02-07T12:58:50.519664+0000 front 2024-02-07T12:58:50.519689+0000 (oldest deadline 2024-02-07T12:59:13.419542+0000) 2024-02-07T12:59:22.369 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:22 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:22.089+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:59:22.369 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:22 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:22.089+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:34.015766+0000 front 2024-02-07T12:58:34.015850+0000 (oldest deadline 2024-02-07T12:58:58.616037+0000) 2024-02-07T12:59:22.369 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:22 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:22.089+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:43.117882+0000 front 2024-02-07T12:58:43.117961+0000 (oldest deadline 2024-02-07T12:59:07.217760+0000) 2024-02-07T12:59:22.369 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:22 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:22.089+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6830 osd.3 since back 2024-02-07T12:58:50.519664+0000 front 2024-02-07T12:58:50.519689+0000 (oldest deadline 2024-02-07T12:59:13.419542+0000) 2024-02-07T12:59:23.618 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:23 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:23.118+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:59:23.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:23 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:23.118+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:34.015766+0000 front 2024-02-07T12:58:34.015850+0000 (oldest deadline 2024-02-07T12:58:58.616037+0000) 2024-02-07T12:59:23.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:23 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:23.118+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:43.117882+0000 front 2024-02-07T12:58:43.117961+0000 (oldest deadline 2024-02-07T12:59:07.217760+0000) 2024-02-07T12:59:23.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:23 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:23.118+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6830 osd.3 since back 2024-02-07T12:58:50.519664+0000 front 2024-02-07T12:58:50.519689+0000 (oldest deadline 2024-02-07T12:59:13.419542+0000) 2024-02-07T12:59:23.844 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:59:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:59:23.555Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=5 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi172.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.172:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi019.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.19:8443: connect: connection refused" 2024-02-07T12:59:23.844 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:59:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:59:23.555Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi019.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.19:8443: connect: connection refused" 2024-02-07T12:59:23.844 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:59:23 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:59:23.556Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi172.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.172:8443: connect: connection refused" 2024-02-07T12:59:24.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:24 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:24.116+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6806 osd.0 since back 2024-02-07T12:58:28.715304+0000 front 2024-02-07T12:58:28.715288+0000 (oldest deadline 2024-02-07T12:58:54.015170+0000) 2024-02-07T12:59:24.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:24 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:24.116+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6814 osd.1 since back 2024-02-07T12:58:34.015766+0000 front 2024-02-07T12:58:34.015850+0000 (oldest deadline 2024-02-07T12:58:58.616037+0000) 2024-02-07T12:59:24.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:24 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:24.116+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6822 osd.2 since back 2024-02-07T12:58:43.117882+0000 front 2024-02-07T12:58:43.117961+0000 (oldest deadline 2024-02-07T12:59:07.217760+0000) 2024-02-07T12:59:24.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:24 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7[91906]: 2024-02-07T12:59:24.116+0000 7f1bad4e4700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.19:6830 osd.3 since back 2024-02-07T12:58:50.519664+0000 front 2024-02-07T12:58:50.519689+0000 (oldest deadline 2024-02-07T12:59:13.419542+0000) 2024-02-07T12:59:25.173 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:24 smithi172 podman[101341]: 2024-02-07 12:59:24.909092287 +0000 UTC m=+5.457932313 container died 75dab2b6606bcc90dbf7e7367bc75136f1dd8d09860d216679761d706aa50a9d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True) 2024-02-07T12:59:25.548 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:25 smithi172 podman[101341]: 2024-02-07 12:59:25.270806509 +0000 UTC m=+5.819646582 container cleanup 75dab2b6606bcc90dbf7e7367bc75136f1dd8d09860d216679761d706aa50a9d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131) 2024-02-07T12:59:25.548 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:25 smithi172 bash[101341]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7 2024-02-07T12:59:25.548 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:59:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:59:25.377Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.19:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.19:8765: connect: connection refused" 2024-02-07T12:59:25.548 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:59:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:59:25.378Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.19:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.19:8765: connect: connection refused" 2024-02-07T12:59:25.548 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:59:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:59:25.380Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.19:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.19:8765: connect: connection refused" 2024-02-07T12:59:25.548 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:59:25 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:59:25.381Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.19:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.19:8765: connect: connection refused" 2024-02-07T12:59:25.869 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:25 smithi172 podman[101370]: 2024-02-07 12:59:25.546895886 +0000 UTC m=+0.633100742 container remove 75dab2b6606bcc90dbf7e7367bc75136f1dd8d09860d216679761d706aa50a9d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True, maintainer=Guillaume Abrioux ) 2024-02-07T12:59:26.235 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:25 smithi172 podman[101462]: 2024-02-07 12:59:25.803165527 +0000 UTC m=+0.018484401 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:59:26.235 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:25 smithi172 podman[101462]: 2024-02-07 12:59:25.922965938 +0000 UTC m=+0.138284804 container create 1de7fc28976147c020859a053482b8eaa6eef792ca472407d780d874a544cf3e (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7-deactivate, org.label-schema.build-date=20240131, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:59:26.235 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:26 smithi172 podman[101462]: 2024-02-07 12:59:26.133556529 +0000 UTC m=+0.348875405 container init 1de7fc28976147c020859a053482b8eaa6eef792ca472407d780d874a544cf3e (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7-deactivate, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2) 2024-02-07T12:59:26.235 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:26 smithi172 podman[101462]: 2024-02-07 12:59:26.136570827 +0000 UTC m=+0.351889690 container start 1de7fc28976147c020859a053482b8eaa6eef792ca472407d780d874a544cf3e (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, io.buildah.version=1.33.2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.license=GPLv2, org.label-schema.build-date=20240131, GIT_CLEAN=True) 2024-02-07T12:59:26.235 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:26 smithi172 podman[101462]: 2024-02-07 12:59:26.233548884 +0000 UTC m=+0.448867750 container attach 1de7fc28976147c020859a053482b8eaa6eef792ca472407d780d874a544cf3e (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7-deactivate, ceph=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240131, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:59:26.619 INFO:journalctl@ceph.osd.7.smithi172.stdout:Feb 07 12:59:26 smithi172 podman[101462]: 2024-02-07 12:59:26.428342349 +0000 UTC m=+0.643661212 container died 1de7fc28976147c020859a053482b8eaa6eef792ca472407d780d874a544cf3e (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-osd-7-deactivate, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131) 2024-02-07T12:59:27.086 DEBUG:teuthology.orchestra.run.smithi172:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@osd.7.service' 2024-02-07T12:59:27.126 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-07T12:59:27.127 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-02-07T12:59:27.127 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-02-07T12:59:27.127 DEBUG:teuthology.orchestra.run.smithi172:> sudo systemctl stop ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@prometheus.a 2024-02-07T12:59:27.368 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:59:27 smithi172 systemd[1]: Stopping Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:59:27.743 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:59:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:59:27.473Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-02-07T12:59:27.743 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:59:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:59:27.473Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-02-07T12:59:27.743 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:59:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:59:27.473Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-02-07T12:59:27.743 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:59:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:59:27.473Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-02-07T12:59:27.743 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:59:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:59:27.473Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-02-07T12:59:27.743 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:59:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:59:27.473Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-02-07T12:59:27.743 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:59:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:59:27.473Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-02-07T12:59:27.743 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:59:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:59:27.473Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-02-07T12:59:27.743 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:59:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:59:27.474Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-02-07T12:59:27.743 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:59:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:59:27.474Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-02-07T12:59:27.744 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:59:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:59:27.474Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-02-07T12:59:27.744 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:59:27 smithi172 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a[63932]: ts=2024-02-07T12:59:27.474Z caller=main.go:1147 level=info msg="See you next time!" 2024-02-07T12:59:27.744 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:59:27 smithi172 podman[101568]: 2024-02-07 12:59:27.484088064 +0000 UTC m=+0.136387509 container died c9b607eea1be415ba6cacefac1bbc72efcc7c587f1ca609bd355be1bcca50d55 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:59:28.101 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:59:27 smithi172 podman[101568]: 2024-02-07 12:59:27.924201254 +0000 UTC m=+0.576500704 container cleanup c9b607eea1be415ba6cacefac1bbc72efcc7c587f1ca609bd355be1bcca50d55 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:59:28.101 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:59:27 smithi172 bash[101568]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a 2024-02-07T12:59:28.350 DEBUG:teuthology.orchestra.run.smithi172:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@prometheus.a.service' 2024-02-07T12:59:28.368 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:59:28 smithi172 podman[101582]: 2024-02-07 12:59:28.10027579 +0000 UTC m=+0.618784851 container remove c9b607eea1be415ba6cacefac1bbc72efcc7c587f1ca609bd355be1bcca50d55 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-02-07T12:59:28.369 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:59:28 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@prometheus.a.service: Deactivated successfully. 2024-02-07T12:59:28.369 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:59:28 smithi172 systemd[1]: Stopped Ceph prometheus.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:59:28.369 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Feb 07 12:59:28 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@prometheus.a.service: Consumed 3.874s CPU time. 2024-02-07T12:59:28.386 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-07T12:59:28.386 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-02-07T12:59:28.387 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 --force --keep-logs 2024-02-07T12:59:30.109 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:59:29 smithi019 systemd[1]: Stopping Ceph node-exporter.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:59:30.392 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:59:30 smithi019 podman[135506]: 2024-02-07 12:59:30.108078668 +0000 UTC m=+0.116131937 container died c4bdcb1c4d49b53ed683f2d70ce8ef68c15a0fee19ecac17295c5c34bb77533d (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-02-07T12:59:30.825 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:59:30 smithi019 podman[135506]: 2024-02-07 12:59:30.556429909 +0000 UTC m=+0.564483222 container cleanup c4bdcb1c4d49b53ed683f2d70ce8ef68c15a0fee19ecac17295c5c34bb77533d (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-02-07T12:59:30.825 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:59:30 smithi019 bash[135506]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a 2024-02-07T12:59:31.096 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:59:30 smithi019 podman[135520]: 2024-02-07 12:59:30.824156743 +0000 UTC m=+0.720360965 container remove c4bdcb1c4d49b53ed683f2d70ce8ef68c15a0fee19ecac17295c5c34bb77533d (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-02-07T12:59:31.096 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:59:30 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-02-07T12:59:31.097 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:59:30 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@node-exporter.a.service: Failed with result 'exit-code'. 2024-02-07T12:59:31.097 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:59:30 smithi019 systemd[1]: Stopped Ceph node-exporter.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:59:31.097 INFO:journalctl@ceph.node-exporter.a.smithi019.stdout:Feb 07 12:59:30 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@node-exporter.a.service: Consumed 3.189s CPU time. 2024-02-07T12:59:31.444 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:59:31 smithi019 systemd[1]: Stopping Ceph alertmanager.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:59:31.444 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:59:31 smithi019 ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a[76557]: ts=2024-02-07T12:59:31.402Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-02-07T12:59:31.704 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:59:31 smithi019 podman[135629]: 2024-02-07 12:59:31.443132979 +0000 UTC m=+0.142666884 container died fec9255f35e7f325c423bf0f247b5f327b09b25c9bdffc61bb052ef40a0052bb (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-02-07T12:59:32.012 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:59:31 smithi019 podman[135629]: 2024-02-07 12:59:31.768771262 +0000 UTC m=+0.468305171 container cleanup fec9255f35e7f325c423bf0f247b5f327b09b25c9bdffc61bb052ef40a0052bb (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-02-07T12:59:32.013 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:59:31 smithi019 bash[135629]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a 2024-02-07T12:59:32.013 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:59:31 smithi019 podman[135643]: 2024-02-07 12:59:31.936343731 +0000 UTC m=+0.488608090 container remove fec9255f35e7f325c423bf0f247b5f327b09b25c9bdffc61bb052ef40a0052bb (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-02-07T12:59:32.013 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:59:32 smithi019 podman[135643]: 2024-02-07 12:59:32.012050516 +0000 UTC m=+0.564314873 volume remove bd14446a59981c254226057a9ae8a5b50255d6a115e3608cbb998b9abd1c9242 2024-02-07T12:59:32.593 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:59:32 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@alertmanager.a.service: Deactivated successfully. 2024-02-07T12:59:32.594 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:59:32 smithi019 systemd[1]: Stopped Ceph alertmanager.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:59:32.594 INFO:journalctl@ceph.alertmanager.a.smithi019.stdout:Feb 07 12:59:32 smithi019 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@alertmanager.a.service: Consumed 1.662s CPU time. 2024-02-07T12:59:55.397 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 --force --keep-logs 2024-02-07T12:59:57.119 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:59:56 smithi172 systemd[1]: Stopping Ceph node-exporter.b for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:59:57.119 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:59:57 smithi172 podman[101965]: 2024-02-07 12:59:57.06027009 +0000 UTC m=+0.107778792 container died 04ea81f79d853bc93a320860fb65f18461b68ebef6cc94f5d7a6fb2cf02f41da (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-02-07T12:59:57.869 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:59:57 smithi172 podman[101965]: 2024-02-07 12:59:57.491501193 +0000 UTC m=+0.539009894 container cleanup 04ea81f79d853bc93a320860fb65f18461b68ebef6cc94f5d7a6fb2cf02f41da (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-02-07T12:59:57.869 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:59:57 smithi172 bash[101965]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b 2024-02-07T12:59:57.869 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:59:57 smithi172 podman[101979]: 2024-02-07 12:59:57.684730741 +0000 UTC m=+0.628964051 container remove 04ea81f79d853bc93a320860fb65f18461b68ebef6cc94f5d7a6fb2cf02f41da (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-02-07T12:59:57.869 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:59:57 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-02-07T12:59:58.366 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:59:58 smithi172 systemd[1]: Stopping Ceph grafana.a for ced2783c-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:59:58.366 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:59:57 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@node-exporter.b.service: Failed with result 'exit-code'. 2024-02-07T12:59:58.366 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:59:57 smithi172 systemd[1]: Stopped Ceph node-exporter.b for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:59:58.366 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Feb 07 12:59:57 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@node-exporter.b.service: Consumed 3.101s CPU time. 2024-02-07T12:59:58.618 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:59:58 smithi172 podman[102097]: 2024-02-07 12:59:58.364846878 +0000 UTC m=+0.117529284 container died aaffac63bc281628aa7d0fce29262aa3651c7c60651205efa0d467922a09b43c (image=quay.io/ceph/ceph-grafana:9.4.12, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, io.buildah.version=1.32.0, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, name=ubi8, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-02-07T12:59:59.094 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:59:58 smithi172 podman[102097]: 2024-02-07 12:59:58.696140456 +0000 UTC m=+0.448822873 container cleanup aaffac63bc281628aa7d0fce29262aa3651c7c60651205efa0d467922a09b43c (image=quay.io/ceph/ceph-grafana:9.4.12, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, vendor=Red Hat, Inc., vcs-type=git, build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, architecture=x86_64, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, io.buildah.version=1.32.0, description=Ceph Grafana Container, release=239.1651231664, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-02-07T12:59:59.094 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:59:58 smithi172 bash[102097]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a 2024-02-07T12:59:59.094 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:59:58 smithi172 podman[102111]: 2024-02-07 12:59:58.863651514 +0000 UTC m=+0.498085548 container remove aaffac63bc281628aa7d0fce29262aa3651c7c60651205efa0d467922a09b43c (image=quay.io/ceph/ceph-grafana:9.4.12, name=ceph-ced2783c-c5b2-11ee-95b6-87774f69a715-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, vendor=Red Hat, Inc., version=8.5, io.openshift.tags=base rhel8, release=239.1651231664, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, description=Ceph Grafana Container) 2024-02-07T12:59:59.094 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:59:58 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-02-07T12:59:59.368 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:59:59 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-02-07T12:59:59.369 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:59:59 smithi172 systemd[1]: Stopped Ceph grafana.a for ced2783c-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:59:59.369 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Feb 07 12:59:59 smithi172 systemd[1]: ceph-ced2783c-c5b2-11ee-95b6-87774f69a715@grafana.a.service: Consumed 9.341s CPU time. 2024-02-07T13:00:11.017 DEBUG:teuthology.orchestra.run.smithi019:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-02-07T13:00:11.054 DEBUG:teuthology.orchestra.run.smithi172:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-02-07T13:00:11.083 INFO:tasks.cephadm:Archiving crash dumps... 2024-02-07T13:00:11.085 DEBUG:teuthology.misc:Transferring archived files from smithi019:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/crash to /home/teuthworker/archive/gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi/7548733/remote/smithi019/crash 2024-02-07T13:00:11.086 DEBUG:teuthology.orchestra.run.smithi019:> sudo tar c -f - -C /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/crash -- . 2024-02-07T13:00:11.143 INFO:teuthology.orchestra.run.smithi019.stderr:tar: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/crash: Cannot open: No such file or directory 2024-02-07T13:00:11.143 INFO:teuthology.orchestra.run.smithi019.stderr:tar: Error is not recoverable: exiting now 2024-02-07T13:00:11.145 DEBUG:teuthology.misc:Transferring archived files from smithi172:/var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/crash to /home/teuthworker/archive/gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi/7548733/remote/smithi172/crash 2024-02-07T13:00:11.145 DEBUG:teuthology.orchestra.run.smithi172:> sudo tar c -f - -C /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/crash -- . 2024-02-07T13:00:11.185 INFO:teuthology.orchestra.run.smithi172.stderr:tar: /var/lib/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/crash: Cannot open: No such file or directory 2024-02-07T13:00:11.185 INFO:teuthology.orchestra.run.smithi172.stderr:tar: Error is not recoverable: exiting now 2024-02-07T13:00:11.186 INFO:tasks.cephadm:Checking cluster log for badness... 2024-02-07T13:00:11.187 DEBUG:teuthology.orchestra.run.smithi019:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-02-07T13:00:11.218 INFO:teuthology.orchestra.run.smithi019.stdout:2024-02-07T12:19:14.252813+0000 mon.a (mon.0) 168 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-02-07T13:00:11.218 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-02-07T13:00:11.218 DEBUG:teuthology.orchestra.run.smithi019:> sudo egrep '\[SEC\]' /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-02-07T13:00:11.288 DEBUG:teuthology.orchestra.run.smithi019:> sudo egrep '\[ERR\]' /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-02-07T13:00:11.360 DEBUG:teuthology.orchestra.run.smithi019:> sudo egrep '\[WRN\]' /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-02-07T13:00:11.430 INFO:teuthology.orchestra.run.smithi019.stdout:2024-02-07T12:19:14.252813+0000 mon.a (mon.0) 168 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-02-07T13:00:11.430 INFO:tasks.cephadm:Compressing logs... 2024-02-07T13:00:11.430 DEBUG:teuthology.orchestra.run.smithi019:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-02-07T13:00:11.473 DEBUG:teuthology.orchestra.run.smithi172:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-02-07T13:00:11.498 INFO:teuthology.orchestra.run.smithi019.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-02-07T13:00:11.498 INFO:teuthology.orchestra.run.smithi019.stderr:: No such file or directory 2024-02-07T13:00:11.498 INFO:teuthology.orchestra.run.smithi019.stderr:gzip -5 --verbose -- /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-mon.a.log 2024-02-07T13:00:11.499 INFO:teuthology.orchestra.run.smithi019.stderr:gzip -5 --verbose -- /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph.audit.log 2024-02-07T13:00:11.499 INFO:teuthology.orchestra.run.smithi019.stderr:/var/log/ceph/cephadm.log: /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-osd.1.log 2024-02-07T13:00:11.499 INFO:teuthology.orchestra.run.smithi019.stderr:/var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-osd.3.log 2024-02-07T13:00:11.499 INFO:teuthology.orchestra.run.smithi019.stderr:gzip -5 --verbose -- /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-mgr.y.log 2024-02-07T13:00:11.507 INFO:teuthology.orchestra.run.smithi019.stderr:/var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-osd.1.log: /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/tcmu-runner.log 2024-02-07T13:00:11.507 INFO:teuthology.orchestra.run.smithi019.stderr:/var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-osd.0.log 2024-02-07T13:00:11.507 INFO:teuthology.orchestra.run.smithi019.stderr:/var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/tcmu-runner.log: gzip -5 --verbose -- /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-client.rgw.foo.smithi019.ciualv.log 2024-02-07T13:00:11.507 INFO:teuthology.orchestra.run.smithi019.stderr: 86.0% -- replaced with /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/tcmu-runner.log.gz 2024-02-07T13:00:11.507 INFO:teuthology.orchestra.run.smithi019.stderr:/var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph.cephadm.log 2024-02-07T13:00:11.507 INFO:teuthology.orchestra.run.smithi019.stderr:/var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-client.rgw.foo.smithi019.ciualv.log: 76.3% -- replaced with /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-client.rgw.foo.smithi019.ciualv.log.gz 2024-02-07T13:00:11.507 INFO:teuthology.orchestra.run.smithi019.stderr:gzip -5 --verbose -- /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph.log 2024-02-07T13:00:11.508 INFO:teuthology.orchestra.run.smithi019.stderr:/var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-osd.2.log 2024-02-07T13:00:11.513 INFO:teuthology.orchestra.run.smithi172.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-02-07T13:00:11.513 INFO:teuthology.orchestra.run.smithi172.stderr:: No such file or directory 2024-02-07T13:00:11.513 INFO:teuthology.orchestra.run.smithi172.stderr:gzip -5 --verbose -- /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-osd.6.log 2024-02-07T13:00:11.513 INFO:teuthology.orchestra.run.smithi172.stderr:gzip -5 --verbose -- /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph.audit.log 2024-02-07T13:00:11.513 INFO:teuthology.orchestra.run.smithi172.stderr:/var/log/ceph/cephadm.log: /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-osd.4.log 2024-02-07T13:00:11.513 INFO:teuthology.orchestra.run.smithi172.stderr:/var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-osd.7.log 2024-02-07T13:00:11.513 INFO:teuthology.orchestra.run.smithi172.stderr:/var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-osd.5.log 2024-02-07T13:00:11.513 INFO:teuthology.orchestra.run.smithi172.stderr:/var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-client.rgw.foo.smithi172.wueguy.log 2024-02-07T13:00:11.513 INFO:teuthology.orchestra.run.smithi172.stderr:/var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-mgr.x.log 2024-02-07T13:00:11.514 INFO:teuthology.orchestra.run.smithi172.stderr:/var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-client.rgw.foo.smithi172.wueguy.log: gzip -5 --verbose -- /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph.cephadm.log 2024-02-07T13:00:11.514 INFO:teuthology.orchestra.run.smithi172.stderr: 76.3% -- replaced with /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-client.rgw.foo.smithi172.wueguy.log.gz 2024-02-07T13:00:11.514 INFO:teuthology.orchestra.run.smithi172.stderr:/var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-mgr.x.log: 91.3% -- replaced with /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph.audit.log.gz 2024-02-07T13:00:11.514 INFO:teuthology.orchestra.run.smithi172.stderr:gzip -5 --verbose -- /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph.log 2024-02-07T13:00:11.514 INFO:teuthology.orchestra.run.smithi172.stderr:/var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-volume.log 2024-02-07T13:00:11.514 INFO:teuthology.orchestra.run.smithi172.stderr:/var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph.log: 95.8% -- replaced with /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph.cephadm.log.gz 2024-02-07T13:00:11.517 INFO:teuthology.orchestra.run.smithi172.stderr: 89.1% -- replaced with /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph.log.gz 2024-02-07T13:00:11.518 INFO:teuthology.orchestra.run.smithi172.stderr:gzip -5 --verbose -- /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-mon.b.log 2024-02-07T13:00:11.518 INFO:teuthology.orchestra.run.smithi172.stderr: 86.5%/var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-volume.log: -- replaced with /var/log/ceph/cephadm.log.gz 2024-02-07T13:00:11.519 INFO:teuthology.orchestra.run.smithi019.stderr:/var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph.log: 97.0% -- replaced with /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph.cephadm.log.gz 2024-02-07T13:00:11.519 INFO:teuthology.orchestra.run.smithi019.stderr: 94.6% -- replaced with /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph.audit.log.gz 2024-02-07T13:00:11.519 INFO:teuthology.orchestra.run.smithi019.stderr: 92.3% -- replaced with /var/log/ceph/cephadm.log.gz 2024-02-07T13:00:11.519 INFO:teuthology.orchestra.run.smithi019.stderr: 93.9% -- replaced with /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph.log.gz 2024-02-07T13:00:11.519 INFO:teuthology.orchestra.run.smithi019.stderr:gzip -5 --verbose -- /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-volume.log 2024-02-07T13:00:11.519 INFO:teuthology.orchestra.run.smithi019.stderr:/var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-mon.c.log 2024-02-07T13:00:11.531 INFO:teuthology.orchestra.run.smithi172.stderr:/var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-mon.b.log: 91.1% -- replaced with /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-volume.log.gz 2024-02-07T13:00:11.533 INFO:teuthology.orchestra.run.smithi019.stderr:/var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-volume.log: /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-mon.c.log: 91.1% -- replaced with /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-volume.log.gz 2024-02-07T13:00:11.642 INFO:teuthology.orchestra.run.smithi172.stderr: 89.7% -- replaced with /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-mgr.x.log.gz 2024-02-07T13:00:12.027 INFO:teuthology.orchestra.run.smithi019.stderr: 88.9% -- replaced with /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-mgr.y.log.gz 2024-02-07T13:00:12.229 INFO:teuthology.orchestra.run.smithi172.stderr: 92.3% -- replaced with /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-mon.b.log.gz 2024-02-07T13:00:12.301 INFO:teuthology.orchestra.run.smithi019.stderr: 92.3% -- replaced with /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-mon.c.log.gz 2024-02-07T13:00:12.972 INFO:teuthology.orchestra.run.smithi172.stderr: 93.1% -- replaced with /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-osd.6.log.gz 2024-02-07T13:00:13.099 INFO:teuthology.orchestra.run.smithi019.stderr: 91.1% -- replaced with /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-mon.a.log.gz 2024-02-07T13:00:13.298 INFO:teuthology.orchestra.run.smithi172.stderr: 93.7% -- replaced with /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-osd.7.log.gz 2024-02-07T13:00:13.314 INFO:teuthology.orchestra.run.smithi019.stderr: 93.3% -- replaced with /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-osd.2.log.gz 2024-02-07T13:00:13.528 INFO:teuthology.orchestra.run.smithi019.stderr: 93.4% -- replaced with /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-osd.1.log.gz 2024-02-07T13:00:13.552 INFO:teuthology.orchestra.run.smithi019.stderr: 93.4% -- replaced with /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-osd.0.log.gz 2024-02-07T13:00:13.603 INFO:teuthology.orchestra.run.smithi172.stderr: 93.3% -- replaced with /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-osd.5.log.gz 2024-02-07T13:00:13.791 INFO:teuthology.orchestra.run.smithi172.stderr: 93.5% -- replaced with /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-osd.4.log.gz 2024-02-07T13:00:13.792 INFO:teuthology.orchestra.run.smithi172.stderr: 2024-02-07T13:00:13.793 INFO:teuthology.orchestra.run.smithi172.stderr:real 0m2.305s 2024-02-07T13:00:13.793 INFO:teuthology.orchestra.run.smithi172.stderr:user 0m8.255s 2024-02-07T13:00:13.793 INFO:teuthology.orchestra.run.smithi172.stderr:sys 0m0.323s 2024-02-07T13:00:14.369 INFO:teuthology.orchestra.run.smithi019.stderr: 93.3% -- replaced with /var/log/ceph/ced2783c-c5b2-11ee-95b6-87774f69a715/ceph-osd.3.log.gz 2024-02-07T13:00:14.371 INFO:teuthology.orchestra.run.smithi019.stderr: 2024-02-07T13:00:14.371 INFO:teuthology.orchestra.run.smithi019.stderr:real 0m2.886s 2024-02-07T13:00:14.371 INFO:teuthology.orchestra.run.smithi019.stderr:user 0m11.351s 2024-02-07T13:00:14.371 INFO:teuthology.orchestra.run.smithi019.stderr:sys 0m0.384s 2024-02-07T13:00:14.372 INFO:tasks.cephadm:Archiving logs... 2024-02-07T13:00:14.372 DEBUG:teuthology.misc:Transferring archived files from smithi019:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi/7548733/remote/smithi019/log 2024-02-07T13:00:14.373 DEBUG:teuthology.orchestra.run.smithi019:> sudo tar c -f - -C /var/log/ceph -- . 2024-02-07T13:00:16.045 DEBUG:teuthology.misc:Transferring archived files from smithi172:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi/7548733/remote/smithi172/log 2024-02-07T13:00:16.046 DEBUG:teuthology.orchestra.run.smithi172:> sudo tar c -f - -C /var/log/ceph -- . 2024-02-07T13:00:17.284 INFO:tasks.cephadm:Removing cluster... 2024-02-07T13:00:17.285 DEBUG:teuthology.orchestra.run.smithi019:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 --force 2024-02-07T13:00:17.577 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ced2783c-c5b2-11ee-95b6-87774f69a715 --force 2024-02-07T13:00:17.871 INFO:tasks.cephadm:Removing cephadm ... 2024-02-07T13:00:17.871 DEBUG:teuthology.orchestra.run.smithi019:> rm -rf /home/ubuntu/cephtest/cephadm 2024-02-07T13:00:17.886 DEBUG:teuthology.orchestra.run.smithi172:> rm -rf /home/ubuntu/cephtest/cephadm 2024-02-07T13:00:17.902 INFO:tasks.cephadm:Teardown complete 2024-02-07T13:00:17.903 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-02-07T13:00:17.913 INFO:teuthology.task.clock:Checking final clock skew... 2024-02-07T13:00:17.913 DEBUG:teuthology.orchestra.run.smithi019:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-02-07T13:00:17.929 DEBUG:teuthology.orchestra.run.smithi172:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-02-07T13:00:17.941 INFO:teuthology.orchestra.run.smithi019.stderr:bash: line 1: ntpq: command not found 2024-02-07T13:00:17.945 INFO:teuthology.orchestra.run.smithi019.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-02-07T13:00:17.945 INFO:teuthology.orchestra.run.smithi019.stdout:=============================================================================== 2024-02-07T13:00:17.945 INFO:teuthology.orchestra.run.smithi019.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 1 -854us[ -842us] +/- 78ms 2024-02-07T13:00:17.945 INFO:teuthology.orchestra.run.smithi019.stdout:^* hv02.front.sepia.ceph.com 3 6 377 0 +181us[ +193us] +/- 54ms 2024-02-07T13:00:17.945 INFO:teuthology.orchestra.run.smithi019.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 62 +555us[ +566us] +/- 73ms 2024-02-07T13:00:17.945 INFO:teuthology.orchestra.run.smithi019.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-02-07T13:00:17.957 INFO:teuthology.orchestra.run.smithi172.stderr:bash: line 1: ntpq: command not found 2024-02-07T13:00:17.961 INFO:teuthology.orchestra.run.smithi172.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-02-07T13:00:17.961 INFO:teuthology.orchestra.run.smithi172.stdout:=============================================================================== 2024-02-07T13:00:17.961 INFO:teuthology.orchestra.run.smithi172.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 64 -866us[ -882us] +/- 77ms 2024-02-07T13:00:17.961 INFO:teuthology.orchestra.run.smithi172.stdout:^* hv02.front.sepia.ceph.com 3 7 377 0 +180us[ +169us] +/- 54ms 2024-02-07T13:00:17.961 INFO:teuthology.orchestra.run.smithi172.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 2 +552us[ +541us] +/- 74ms 2024-02-07T13:00:17.961 INFO:teuthology.orchestra.run.smithi172.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-02-07T13:00:17.962 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-02-07T13:00:17.973 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-02-07T13:00:17.974 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-02-07T13:00:17.983 DEBUG:teuthology.orchestra.run.smithi019:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-02-07T13:00:18.082 DEBUG:teuthology.orchestra.run.smithi172:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-02-07T13:00:18.162 DEBUG:teuthology.orchestra.run.smithi019:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-02-07T13:00:18.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-02-07T13:00:18.193 DEBUG:teuthology.orchestra.run.smithi172:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-02-07T13:00:18.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-02-07T13:00:18.225 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-02-07T13:00:18.235 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-02-07T13:00:18.259 INFO:teuthology.task.internal:Duration was 2843.096542 seconds 2024-02-07T13:00:18.260 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-02-07T13:00:18.269 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-02-07T13:00:18.270 DEBUG:teuthology.orchestra.run.smithi019:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-02-07T13:00:18.272 DEBUG:teuthology.orchestra.run.smithi172:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-02-07T13:00:18.313 INFO:teuthology.orchestra.run.smithi019.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-02-07T13:00:18.315 INFO:teuthology.orchestra.run.smithi172.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-02-07T13:00:18.657 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-02-07T13:00:18.657 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi019.front.sepia.ceph.com 2024-02-07T13:00:18.658 DEBUG:teuthology.orchestra.run.smithi019:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-02-07T13:00:18.695 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi172.front.sepia.ceph.com 2024-02-07T13:00:18.695 DEBUG:teuthology.orchestra.run.smithi172:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-02-07T13:00:18.720 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-02-07T13:00:18.720 DEBUG:teuthology.orchestra.run.smithi019:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-02-07T13:00:18.737 DEBUG:teuthology.orchestra.run.smithi172:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-02-07T13:00:18.788 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-02-07T13:00:18.789 DEBUG:teuthology.orchestra.run.smithi019:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-02-07T13:00:18.807 DEBUG:teuthology.orchestra.run.smithi172:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-02-07T13:00:19.465 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-02-07T13:00:19.475 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-02-07T13:00:19.476 DEBUG:teuthology.orchestra.run.smithi019:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-02-07T13:00:19.503 DEBUG:teuthology.orchestra.run.smithi172:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-02-07T13:00:19.530 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-02-07T13:00:19.542 DEBUG:teuthology.orchestra.run.smithi019:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-02-07T13:00:19.545 DEBUG:teuthology.orchestra.run.smithi172:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-02-07T13:00:19.568 INFO:teuthology.orchestra.run.smithi019.stdout:kernel.core_pattern = core 2024-02-07T13:00:19.596 INFO:teuthology.orchestra.run.smithi172.stdout:kernel.core_pattern = core 2024-02-07T13:00:19.639 DEBUG:teuthology.orchestra.run.smithi019:> test -e /home/ubuntu/cephtest/archive/coredump 2024-02-07T13:00:19.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-02-07T13:00:19.653 DEBUG:teuthology.orchestra.run.smithi172:> test -e /home/ubuntu/cephtest/archive/coredump 2024-02-07T13:00:19.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-02-07T13:00:19.693 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-02-07T13:00:19.704 INFO:teuthology.task.internal:Transferring archived files... 2024-02-07T13:00:19.705 DEBUG:teuthology.misc:Transferring archived files from smithi019:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi/7548733/remote/smithi019 2024-02-07T13:00:19.705 DEBUG:teuthology.orchestra.run.smithi019:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-02-07T13:00:19.762 DEBUG:teuthology.misc:Transferring archived files from smithi172:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi/7548733/remote/smithi172 2024-02-07T13:00:19.762 DEBUG:teuthology.orchestra.run.smithi172:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-02-07T13:00:19.815 INFO:teuthology.task.internal:Removing archive directory... 2024-02-07T13:00:19.815 DEBUG:teuthology.orchestra.run.smithi019:> rm -rf -- /home/ubuntu/cephtest/archive 2024-02-07T13:00:19.817 DEBUG:teuthology.orchestra.run.smithi172:> rm -rf -- /home/ubuntu/cephtest/archive 2024-02-07T13:00:19.857 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-02-07T13:00:19.871 INFO:teuthology.task.internal:Not uploading archives. 2024-02-07T13:00:19.871 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-02-07T13:00:19.884 INFO:teuthology.task.internal:Tidying up after the test... 2024-02-07T13:00:19.884 DEBUG:teuthology.orchestra.run.smithi019:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-02-07T13:00:19.887 DEBUG:teuthology.orchestra.run.smithi172:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-02-07T13:00:19.899 INFO:teuthology.orchestra.run.smithi019.stdout: 262166 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 7 13:00 /home/ubuntu/cephtest 2024-02-07T13:00:19.913 INFO:teuthology.orchestra.run.smithi172.stdout: 262166 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 7 13:00 /home/ubuntu/cephtest 2024-02-07T13:00:19.914 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-02-07T13:00:19.927 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-02-07T13:00:19.952 INFO:teuthology.nuke:Checking targets against current locks 2024-02-07T13:00:19.978 DEBUG:teuthology.nuke:shortname: smithi019 2024-02-07T13:00:19.978 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-02-07T13:00:20.002 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi019.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi/7548733', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-02-07 12:04:22.370021', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPwFE8Culvu+H8Do8d2TM4Slqxtg4MbUMrg206NbiNC7YeWjtzomiadrQcQ6dXFYEm5ug56l+DV6sOh9A8kd1wU='} 2024-02-07T13:00:20.007 DEBUG:teuthology.nuke:shortname: smithi172 2024-02-07T13:00:20.007 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-02-07T13:00:20.034 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi172.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi/7548733', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-02-07 12:04:22.371828', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFWm+D0M+O6W7kbCyB1ja37/KkmK5mzxLwfjNocn+bQhp3vsJJKP9RHtk4u1YFJWWddksiKfNuLyjszgr90E7J8='} 2024-02-07T13:00:20.059 INFO:teuthology.orchestra.console.smithi019:Power off 2024-02-07T13:00:20.060 DEBUG:teuthology.orchestra.console.smithi019:pexpect command: ipmitool -H smithi019.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-02-07T13:00:20.087 DEBUG:teuthology.orchestra.console.smithi019:power off output: Chassis Power Control: Down/Off 2024-02-07T13:00:20.087 DEBUG:teuthology.orchestra.console.smithi019:pexpect command: ipmitool -H smithi019.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-02-07T13:00:20.106 INFO:teuthology.orchestra.console.smithi172:Power off 2024-02-07T13:00:20.107 DEBUG:teuthology.orchestra.console.smithi172:pexpect command: ipmitool -H smithi172.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-02-07T13:00:20.129 DEBUG:teuthology.orchestra.console.smithi172:power off output: Chassis Power Control: Down/Off 2024-02-07T13:00:20.129 DEBUG:teuthology.orchestra.console.smithi172:pexpect command: ipmitool -H smithi172.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-02-07T13:00:20.138 DEBUG:teuthology.orchestra.console.smithi019:check power output: Chassis Power is on 2024-02-07T13:00:20.153 DEBUG:teuthology.orchestra.console.smithi172:check power output: Chassis Power is on 2024-02-07T13:00:24.139 DEBUG:teuthology.orchestra.console.smithi019:pexpect command: ipmitool -H smithi019.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-02-07T13:00:24.154 DEBUG:teuthology.orchestra.console.smithi172:pexpect command: ipmitool -H smithi172.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-02-07T13:00:24.249 DEBUG:teuthology.orchestra.console.smithi019:check power output: Chassis Power is on 2024-02-07T13:00:24.264 DEBUG:teuthology.orchestra.console.smithi172:check power output: Chassis Power is on 2024-02-07T13:00:28.252 DEBUG:teuthology.orchestra.console.smithi019:pexpect command: ipmitool -H smithi019.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-02-07T13:00:28.265 DEBUG:teuthology.orchestra.console.smithi172:pexpect command: ipmitool -H smithi172.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-02-07T13:00:28.362 DEBUG:teuthology.orchestra.console.smithi019:check power output: Chassis Power is on 2024-02-07T13:00:28.374 DEBUG:teuthology.orchestra.console.smithi172:check power output: Chassis Power is off 2024-02-07T13:00:28.474 INFO:teuthology.orchestra.console.smithi172:Power off completed 2024-02-07T13:00:32.364 DEBUG:teuthology.orchestra.console.smithi019:pexpect command: ipmitool -H smithi019.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-02-07T13:00:32.473 DEBUG:teuthology.orchestra.console.smithi019:check power output: Chassis Power is off 2024-02-07T13:00:32.574 INFO:teuthology.orchestra.console.smithi019:Power off completed 2024-02-07T13:00:32.678 INFO:teuthology.run:Summary data: description: orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} duration: 2843.09654211998 failure_reason: '"2024-02-07T12:19:14.252813+0000 mon.a (mon.0) 168 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,c (MON_DOWN)" in cluster log' owner: scheduled_gabrioux@teuthology success: false 2024-02-07T13:00:32.678 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-02-07T13:00:32.758 INFO:teuthology.run:FAIL